Merge pull request #85 from CCBR/feature/yues_fork

Feature/yues fork

.test/contrasts.tinytest.tsv

@@ -0,0 +1,2 @@
+condition1	condition2
+53_H3K4me3	HN6_H3K4me3

.test/samples.tinytest.tsv

@@ -0,0 +1,4 @@
+sampleName	replicateNumber	isControl	controlName	controlReplicateNumber	path_to_R1	path_to_R2
+53_H3K4me3	1	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/53_H3K4me3_1.R1.fastq.gz	PIPELINE_HOME/.test/53_H3K4me3_1.R2.fastq.gz
+HN6_H3K4me3	1	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/HN6_H3K4me3_1.R1.fastq.gz	PIPELINE_HOME/.test/HN6_H3K4me3_1.R2.fastq.gz
+HN6_IgG_rabbit_negative_control	1	Y	-	-	PIPELINE_HOME/.test/HN6_IgG_rabbit_negative_control_1.R1.fastq.gz	PIPELINE_HOME/.test/HN6_IgG_rabbit_negative_control_1.R2.fastq.gz

carlisle

@@ -21,17 +21,9 @@ SCRIPTBASENAME=$(readlink -f $(basename $0))
 
 #define cluster, partitions dependent on host
 hostID=`echo $HOSTNAME`
-if [[ $hostID == "biowulf.nih.gov" ]]; then
-  BUYINPARTITIONS=$(bash <(curl -s https://raw.githubusercontent.com/CCBR/Tools/master/Biowulf/get_buyin_partition_list.bash 2>/dev/null))
-  PARTITIONS="norm,ccr"
-  cluster_specific_yaml="cluster_biowulf.yaml"
-  tools_specific_yaml="tools_biowulf.yaml"
-  #if [ $BUYINPARTITIONS ];then PARTITIONS="norm,$BUYINPARTITIONS";fi
-elif [[ $hostID == "biowulf8.nih.gov" ]]; then
-  PARTITIONS="rhel8"
-  cluster_specific_yaml="cluster_rhel8.yaml"
-  tools_specific_yaml="tools_rhel8.yaml"
-fi
+PARTITIONS="norm,ccr"
+cluster_specific_yaml="cluster_biowulf.yaml"
+tools_specific_yaml="tools_biowulf.yaml"
 
 # essential files
 # these are relative to the workflows' base folder
@@ -252,6 +244,14 @@ function testrun() {
   dryrun
 }
 
+function tinytestrun() {
+  check_essential_files
+  sed -e "s/PIPELINE_HOME/${PIPELINE_HOME//\//\\/}/g" ${PIPELINE_HOME}/.test/samples.tinytest.tsv > $WORKDIR/config/samples.tsv
+  cp ${PIPELINE_HOME}/.test/contrasts.tinytest.tsv $WORKDIR/config/contrasts.tsv
+  check_essential_files
+  dryrun
+}
+
 function runslurm() {
 # Submit the execution of the pipeline to the biowulf job scheduler (slurm)
   runcheck
@@ -430,6 +430,7 @@ function main(){
     runlocal) runlocal && exit 0;;
     reset) reset && exit 0;;
     testrun) testrun && exit 0;;
+    tinytestrun) tinytestrun && exit 0;;          # hidden option for debugging
     dry) dryrun && exit 0;;                      # hidden option
     local) runlocal && exit 0;;                  # hidden option
     reconfig) reconfig && exit 0;;               # hidden option for debugging
@@ -441,4 +442,4 @@ function main(){
 }
 
 # call the main function
-main "$@"
+main "$@"

resources/cluster_biowulf.yaml

@@ -48,14 +48,28 @@ gopeaks_broad:
 ###################################################################
 findMotif:
     mem: 10g
+    time: 02-00:00:00
     threads: 6
 ###################################################################
 rose:
-    mem: 32g
+    mem: 64g
     threads: 2
-    time: 00-06:00:00
+    time: 00-12:00:00
 ###################################################################
 go_enrichment:
     time: 02-00:00:00
     mem: 10g
-###################################################################
+###################################################################
+deeptools_bw:
+    mem: 30g
+    time: 00-10:00:00
+###################################################################
+deeptools_mat:
+    mem: 30g
+    threads: 16
+    time: 01-00:00:00
+###################################################################
+deeptools_heatmap:
+    mem: 30g
+    time: 01-00:00:00
+###################################################################

resources/tools_biowulf.yaml

@@ -1,20 +1,21 @@
-bedtools: "bedtools/2.30.0"
-bedops: "bedops/2.4.40"
-bowtie2: "bowtie/2-2.4.2"
-cutadapt: "cutadapt/1.18"
-fastqc: "fastqc/0.11.9"
-fastq_screen: "fastq_screen/0.15.2"
-fastxtoolkit: "fastxtoolkit/0.0.14"
-homer: "homer/4.11.1"
-macs2: "macs/2.2.7.1"
-multiqc: "multiqc/1.14"
-perl: "perl/5.34"
-picard: "picard/2.26.9"
-python3: "python/3.7"
-R: "R/4.2.2"
-rose: "ROSE/1.3.1"
-samtools: "samtools/1.15"
-seacr: "seacr/1.4-beta.2"
-ucsc: "ucsc/407"
-gopeaks: "/data/CCBR_Pipeliner/Pipelines/gopeaks/gopeaks"
-fastq_val: "/data/CCBR_Pipeliner/iCLIP/bin/fastQValidator"
+bedtools: "bedtools/2.30.0"
+bedops: "bedops/2.4.41"
+bowtie2: "bowtie/2-2.4.5"
+cutadapt: "cutadapt/4.0"
+deeptools: "deeptools/3.5.1"
+fastqc: "fastqc/0.11.9"
+fastq_screen: "fastq_screen/0.15.2"
+fastxtoolkit: "fastxtoolkit/0.0.14"
+homer: "homer/4.11.1"
+macs2: "macs/2.2.7.1"
+multiqc: "multiqc/1.14"
+perl: "perl/5.34"
+picard: "picard/2.27.3"
+python3: "python/3.9"
+R: "R/4.2.2"
+rose: "ROSE/1.3.1"
+samtools: "samtools/1.15"
+seacr: "SEACR/1.4-beta.2"
+ucsc: "ucsc/445"
+gopeaks: "/data/CCBR_Pipeliner/Pipelines/gopeaks/gopeaks"
+fastq_val: "/data/CCBR_Pipeliner/bin/fastQValidator"

workflow/Snakefile

@@ -14,6 +14,29 @@ def run_library_norm(wildcards):
         files.extend(lib)
     return files
 
+def run_deeptools_bw(wildcards):
+    files=[]
+    b1=expand(join(RESULTSDIR,"deeptools","clean","{replicate}.{dupstatus}.clean.bam"),replicate=REPLICATES,dupstatus=DUPSTATUS)
+    files.extend(b1)
+    b2=expand(join(RESULTSDIR,"deeptools","clean","{replicate}.{dupstatus}.clean.bam.bai"),replicate=REPLICATES,dupstatus=DUPSTATUS)
+    files.extend(b2)
+    b3=expand(join(RESULTSDIR,"deeptools","clean","{replicate}.{dupstatus}.clean.bigwig"),replicate=REPLICATES,dupstatus=DUPSTATUS)
+    files.extend(b3)
+    return files
+
+def run_deeptools_heatmap(wildcards):
+    files=[]
+    GROUP = [a+b for a in TREATMENTS+["all_samples"] for b in ["", ".prot"]]
+    p1=expand(join(RESULTSDIR,"deeptools","{group}.{dupstatus}.metagene_heatmap.pdf"),group=GROUP,dupstatus=DUPSTATUS)
+    files.extend(p1)
+    p2=expand(join(RESULTSDIR,"deeptools","{group}.{dupstatus}.TSS_heatmap.pdf"),group=GROUP,dupstatus=DUPSTATUS)
+    files.extend(p2)
+    p3=expand(join(RESULTSDIR,"deeptools","{group}.{dupstatus}.metagene_profile.pdf"),group=GROUP,dupstatus=DUPSTATUS)
+    files.extend(p3)
+    p4=expand(join(RESULTSDIR,"deeptools","{group}.{dupstatus}.TSS_profile.pdf"),group=GROUP,dupstatus=DUPSTATUS)
+    files.extend(p4)
+    return files
+
 def run_macs2(wildcards):
     files=[]
     if "macs2_narrow" in PEAKTYPE:
@@ -60,13 +83,14 @@ def run_qc(wildcards):
     files=[]
     if ("gopeaks_narrow" in PEAKTYPE) or ("gopeaks_broad" in PEAKTYPE):
         q=expand(join(RESULTSDIR,'qc', 'multiqc_report_{qthresholds}.html'),qthresholds=QTRESHOLDS)
+        files.extend(q)
     else:
         q=join(RESULTSDIR,'qc', 'multiqc_report.html')
-    files.extend(q)
+        files.append(q)
 
     if ("SPIKEIN" == NORM_METHOD) and ("ecoli" == spikein_genome):
         html=join(RESULTSDIR,'qc',"spikein_qc_report.html"),
-        files.extend(html)
+        files.append(html)
     return files
 
 def run_contrasts(wildcards):
@@ -167,30 +191,36 @@ rule all:
         # manifests
         unpack(run_pipe_prep),
 
-        # ALIGN / create_library_norm_scales
-        unpack(run_library_norm),
+        # ALIGN / deeptools_bw
+        unpack(run_deeptools_bw),
 
-        # ALIGN / alignstats
-        expand(join(RESULTSDIR,"alignment_stats","{replicate}.alignment_stats.yaml"),replicate=REPLICATES),
+        # ALIGN / deeptools_heatmap
+        unpack(run_deeptools_heatmap),
+
+        # # ALIGN / create_library_norm_scales
+        # unpack(run_library_norm),
+
+        # # ALIGN / alignstats
+        # expand(join(RESULTSDIR,"alignment_stats","{replicate}.alignment_stats.yaml"),replicate=REPLICATES),
 
-        # ALIGN / gather_alignstats
-        join(RESULTSDIR,"alignment_stats","alignment_stats.tsv"),
+        # # ALIGN / gather_alignstats
+        # join(RESULTSDIR,"alignment_stats","alignment_stats.tsv"),
 
-        # PEAKCALLS / macs2_narrow, macs2_broad, seacr_stringent, seacr_relaxed, gopeaks_narrow, gopeaks_broad
-        unpack(run_macs2),
-        unpack(run_seacr),
-        unpack(run_gopeaks),
+        # # PEAKCALLS / macs2_narrow, macs2_broad, seacr_stringent, seacr_relaxed, gopeaks_narrow, gopeaks_broad
+        # unpack(run_macs2),
+        # unpack(run_seacr),
+        # unpack(run_gopeaks),
 
-        # QC
-        unpack(run_qc),
+        # # QC
+        # unpack(run_qc),
 
-        # DIFF / create_contrast_data_files
-        unpack(run_contrasts),
+        # # DIFF / create_contrast_data_files
+        # unpack(run_contrasts),
 
-        # ANNOTATION / findMotif, rose, create_contrast_peakcaller_files, go_enrichment
-        unpack(get_motifs),
-        unpack(get_rose),
-        unpack(get_enrichment)
+        # # ANNOTATION / findMotif, rose, create_contrast_peakcaller_files, go_enrichment
+        # unpack(get_motifs),
+        # unpack(get_rose),
+        # unpack(get_enrichment)
 """
         ##########################################
         ### intermediate files
@@ -229,4 +259,13 @@ rule all:
         expand(join(RESULTSDIR,"fragments","{replicate}.{dupstatus}.fragments.bed"),replicate=REPLICATES,dupstatus=DUPSTATUS),
         expand(join(RESULTSDIR,"bedgraph","{replicate}.{dupstatus}.bedgraph"),replicate=REPLICATES,dupstatus=DUPSTATUS),
         expand(join(RESULTSDIR,"bigwig","{replicate}.{dupstatus}.bigwig"),replicate=REPLICATES,dupstatus=DUPSTATUS),
+
+        # ALIGN / deeptools_heatmap
+        expand(join(RESULTSDIR,"deeptools","clean", "{sample}.{dupstatus}.deeptools_prep"), sample=TREATMENTS, dupstatus=DUPSTATUS),
+        expand(join(RESULTSDIR,"deeptools","clean", "{sample}.prot.{dupstatus}.deeptools_prep"), sample=TREATMENTS, dupstatus=DUPSTATUS),
+        expand(join(RESULTSDIR,"deeptools","clean", "all_samples.{dupstatus}.deeptools_prep"), dupstatus=DUPSTATUS),
+        expand(join(RESULTSDIR,"deeptools","clean", "all_samples.prot.{dupstatus}.deeptools_prep"), dupstatus=DUPSTATUS),
+        expand(join(RESULTSDIR,"deeptools","clean", "{group}.{dupstatus}.metagene.mat.gz"),group=[a+b for a in TREATMENTS+["all_samples"] for b in ["", ".prot"]],dupstatus=DUPSTATUS),
+        expand(join(RESULTSDIR,"deeptools","clean", "{group}.{dupstatus}.TSS.mat.gz"),group=[a+b for a in TREATMENTS+["all_samples"] for b in ["", ".prot"]],dupstatus=DUPSTATUS),
+        expand(join(RESULTSDIR,"deeptools","clean", "{group}.{dupstatus}.geneinfo.bed"),group=[a+b for a in TREATMENTS+["all_samples"] for b in ["", ".prot"]],dupstatus=DUPSTATUS),
 """