Merge pull request #70 from icgc-argo-workflows/[email protected]

[release]
icgc-argo-workflows · Jun 29, 2023 · 8e0ee44 · 8e0ee44
2 parents 5677186 + e41f16c
commit 8e0ee44
Show file tree

Hide file tree

Showing 17 changed files with 100 additions and 40 deletions.
diff --git a/dna-seq-alignment/main.nf b/dna-seq-alignment/main.nf
@@ -1,7 +1,7 @@
 #!/usr/bin/env nextflow
 nextflow.enable.dsl=2
 name = 'dna-seq-alignment'
-version = '1.9.1'
+version = '1.9.2'
 
 /*
 ========================================================================================
@@ -126,13 +126,18 @@ params.file_info_tsv = "NO_FILE3"
 params.extra_info_tsv = "NO_FILE4"
 params.sequencing_files = []
 
+// song/score setting
 params.song_url = ""
-params.song_container = ""
-params.song_container_version = ""
+params.song_container = "ghcr.io/overture-stack/song-client"
+params.song_container_version = "5.0.2"
 params.score_url = ""
-params.score_container = ""
-params.score_container_version = ""
+params.score_container = "ghcr.io/overture-stack/score"
+params.score_container_version = "5.9.0"
+params.score_mem = 20
+params.score_cpus = 8
+params.score_force = false
 params.api_token = ""
+
 params.download = [:]
 params.seqDataToLaneBam = [:]
 params.bwaMemAligner = [:]
@@ -147,16 +152,19 @@ params.gatkCollectOxogMetrics = [:]
 
 
 download_params = [
-    'cpus': params.cpus,
-    'mem': params.mem,
     'max_retries': params.max_retries,
     'first_retry_wait_time': params.first_retry_wait_time,
     'song_url': params.song_url,
     'song_container': params.song_container,
     'song_container_version': params.song_container_version,
+    'song_cpus': params.cpus,
+    'song_mem': params.mem,
     'score_url': params.score_url,
     'score_container': params.score_container,
-    'score_container_version': params.score_container_version,    
+    'score_container_version': params.score_container_version,
+    'score_cpus' : params.score_cpus,
+    'score_mem' : params.score_mem,
+    'score_transport_mem' : params.score_mem,        
     'api_token': params.api_token,
     *:(params.download ?: [:])
 ]
@@ -214,29 +222,37 @@ payloadGenDnaSeqQc_params = [
 uploadAlignment_params = [
     'max_retries': params.max_retries,
     'first_retry_wait_time': params.first_retry_wait_time,
-    'cpus': params.cpus,
-    'mem': params.mem,
     'song_url': params.song_url,
     'song_container': params.song_container,
     'song_container_version': params.song_container_version,
+    'song_cpus': params.cpus,
+    'song_mem': params.mem,
     'score_url': params.score_url,
     'score_container': params.score_container,
     'score_container_version': params.score_container_version,
+    'score_force' : params.score_force,
+    'score_cpus' : params.score_cpus,
+    'score_mem' : params.score_mem,
+    'score_transport_mem' : params.score_mem,
     'api_token': params.api_token,
     *:(params.uploadAlignment ?: [:])
 ]
 
 uploadQc_params = [
     'max_retries': params.max_retries,
     'first_retry_wait_time': params.first_retry_wait_time,
-    'cpus': params.cpus,
-    'mem': params.mem,
     'song_url': params.song_url,
     'song_container': params.song_container,
     'song_container_version': params.song_container_version,
+    'song_cpus': params.cpus,
+    'song_mem': params.mem,
     'score_url': params.score_url,
     'score_container': params.score_container,
     'score_container_version': params.score_container_version,
+    'score_force' : params.score_force,
+    'score_cpus' : params.score_cpus,
+    'score_mem' : params.score_mem,
+    'score_transport_mem' : params.score_mem,
     'api_token': params.api_token,
     *:(params.uploadQc ?: [:])
 ]
@@ -251,7 +267,7 @@ gatkCollectOxogMetrics_params = [
 
 
 // Include all modules and pass params
-include { SongScoreDownload as dnld } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.8.0/main.nf' params(download_params)
+include { SongScoreDownload as dnld } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf' params(download_params)
 include { seqDataToLaneBam as toLaneBam } from "./modules/raw.githubusercontent.com/icgc-argo/dna-seq-processing-tools/seq-data-to-lane-bam.0.3.3.0/tools/seq-data-to-lane-bam/seq-data-to-lane-bam.nf" params(seqDataToLaneBam_params)
 include { bwaMemAligner } from "./wfpr_modules/github.com/icgc-argo/dna-seq-processing-tools/[email protected]/main.nf" params(bwaMemAligner_params)
 include { readGroupUBamQC as rgQC } from "./modules/raw.githubusercontent.com/icgc-argo/data-qc-tools-and-wfs/read-group-ubam-qc.0.1.2.0/tools/read-group-ubam-qc/read-group-ubam-qc.nf" params(readGroupUBamQC_params)
@@ -263,8 +279,8 @@ include { payloadGenDnaSeqQc as pGenDnaSeqQc } from "./wfpr_modules/github.com/i
 include { gatkSplitIntervals as splitItvls; getSecondaryFiles as getSIIdx } from "./modules/raw.githubusercontent.com/icgc-argo/gatk-tools/gatk-split-intervals.4.1.4.1-1.0/tools/gatk-split-intervals/gatk-split-intervals"
 include { metadataParser as mParser } from "./modules/raw.githubusercontent.com/icgc-argo/data-processing-utility-tools/metadata-parser.0.1.0.0/tools/metadata-parser/metadata-parser.nf"
 include { gatkCollectOxogMetrics as oxog; getOxogSecondaryFiles; gatherOxogMetrics as gatherOM } from "./modules/raw.githubusercontent.com/icgc-argo/gatk-tools/gatk-collect-oxog-metrics.4.1.8.0-3.0/tools/gatk-collect-oxog-metrics/gatk-collect-oxog-metrics" params(gatkCollectOxogMetrics_params)
-include { SongScoreUpload as upAln } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/[email protected].0/main.nf' params(uploadAlignment_params)
-include { SongScoreUpload as upQc } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/[email protected].0/main.nf' params(uploadQc_params)
+include { SongScoreUpload as upAln } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/[email protected].3/main.nf' params(uploadAlignment_params)
+include { SongScoreUpload as upQc } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/[email protected].3/main.nf' params(uploadQc_params)
 include { cleanupWorkdir as cleanup } from './modules/raw.githubusercontent.com/icgc-argo/nextflow-data-processing-utility-tools/2.3.0/process/cleanup-workdir'
 include { payloadGenSeqExperiment as pGenExp } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/[email protected]/main.nf'
 

diff --git a/dna-seq-alignment/pkg.json b/dna-seq-alignment/pkg.json
@@ -1,6 +1,6 @@
 {
     "name": "dna-seq-alignment",
-    "version": "1.9.1",
+    "version": "1.9.2",
     "description": "ICGC ARGO DNA Seq Processing Workflow",
     "main": "main.nf",
     "deprecated": false,
@@ -14,14 +14,14 @@
         "url": "https://github.com/icgc-argo-workflows/dna-seq-processing-wfs.git"
     },
     "dependencies": [
-      "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.8.0",
-      "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/[email protected].0",
-      "github.com/icgc-argo/dna-seq-processing-tools/[email protected]",
-      "github.com/icgc-argo/dna-seq-processing-tools/[email protected]",
-      "github.com/icgc-argo/data-processing-utility-tools/[email protected]",
-      "github.com/icgc-argo/data-processing-utility-tools/[email protected]",
-      "github.com/icgc-argo/data-processing-utility-tools/[email protected]",
-      "github.com/icgc-argo/data-processing-utility-tools/[email protected]"
+        "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0",
+        "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/[email protected].3",
+        "github.com/icgc-argo/dna-seq-processing-tools/[email protected]",
+        "github.com/icgc-argo/dna-seq-processing-tools/[email protected]",
+        "github.com/icgc-argo/data-processing-utility-tools/[email protected]",
+        "github.com/icgc-argo/data-processing-utility-tools/[email protected]",
+        "github.com/icgc-argo/data-processing-utility-tools/[email protected]",
+        "github.com/icgc-argo/data-processing-utility-tools/[email protected]"
     ],
     "devDependencies": [],
     "contributors": [

diff --git a/dna-seq-alignment/tests/local-test-bam.json b/dna-seq-alignment/tests/local-test-bam.json
@@ -13,7 +13,9 @@
     "song_mem": 2,
     "score_url": "https://submission-score.rdpc-qa.cancercollaboratory.org",
     "score_cpus": 3,
-    "score_mem": 8
+    "score_mem": 8,
+    "score_transport_mem" : 8,
+    "score_container_version": "latest"
   },
   "seqDataToLaneBam": {
     "cpus": 2,
@@ -51,12 +53,15 @@
     "song_cpus": 2,
     "song_mem": 2,
     "score_cpus": 3,
-    "score_mem": 8
+    "score_mem": 8,
+    "score_transport_mem" : 8,
+    "score_force": true
   },
   "uploadQc": {
     "song_cpus": 1,
     "song_mem": 1,
     "score_cpus": 1,
-    "score_mem": 1
+    "score_mem": 4,
+    "score_transport_mem" : 4
   }
 }
diff --git a/[email protected]/local_modules/score-download.nf → [email protected]/local_modules/score-download.nf b/[email protected]/local_modules/score-download.nf → [email protected]/local_modules/score-download.nf
@@ -41,6 +41,12 @@ process scoreDownload {
     label "scoreDownload"
     tag "${analysis_id}"
 
+    if (workflow.containerEngine == "singularity") {
+        containerOptions "--bind \$(pwd):/score-client/logs"
+    } else if (workflow.containerEngine == "docker") {
+        containerOptions "-v \$(pwd):/score-client/logs"
+    }
+
     input:
         path analysis
         val study_id

diff --git a/[email protected]/local_modules/song-get-analysis.nf → [email protected]/local_modules/song-get-analysis.nf b/[email protected]/local_modules/song-get-analysis.nf → [email protected]/local_modules/song-get-analysis.nf
@@ -38,6 +38,12 @@ process songGetAnalysis {
 
     tag "${analysis_id}"
 
+    if (workflow.containerEngine == "singularity") {
+        containerOptions "--bind \$(pwd):/song-client/logs"
+    } else if (workflow.containerEngine == "docker") {
+        containerOptions "-v \$(pwd):/song-client/logs"
+    }
+
     input:
         val study_id
         val analysis_id

diff --git a/...y-tools/[email protected]/main.nf → ...y-tools/[email protected]/main.nf b/...y-tools/[email protected]/main.nf → ...y-tools/[email protected]/main.nf
@@ -21,7 +21,7 @@
 */
 
 nextflow.enable.dsl = 2
-version = '2.8.0'
+version = '2.9.0'
 
 // universal params go here, change default value as needed
 params.publish_dir = ""  // set to empty string will disable publishDir

diff --git a/[email protected]/nextflow.config → [email protected]/nextflow.config b/[email protected]/nextflow.config → [email protected]/nextflow.config
diff --git a/...-tools/[email protected]/pkg.json → ...-tools/[email protected]/pkg.json b/...-tools/[email protected]/pkg.json → ...-tools/[email protected]/pkg.json
@@ -1,6 +1,6 @@
 {
     "name": "song-score-download",
-    "version": "2.8.0",
+    "version": "2.9.0",
     "description": "SONG/SCORE download",
     "main": "main.nf",
     "deprecated": false,

diff --git a/...ls/[email protected]/wfpr_modules → ...ls/[email protected]/wfpr_modules b/...ls/[email protected]/wfpr_modules → ...ls/[email protected]/wfpr_modules
diff --git a/[email protected]/local_modules/score-upload.nf → [email protected]/local_modules/score-upload.nf b/[email protected]/local_modules/score-upload.nf → [email protected]/local_modules/score-upload.nf
@@ -10,12 +10,12 @@ params.first_retry_wait_time = 1  // in seconds
 
 // required params w/ default
 params.container = "ghcr.io/overture-stack/score"
-params.container_version = "5.8.1"
+params.container_version = "latest"
 params.transport_mem = 2 // Transport memory is in number of GBs
 
 // optional if secret mounted from pod else required
 params.api_token = "" // song/score API token for download process
-
+params.score_force = false
 // required params, no default
 // --song_url         song url for download process
 // --score_url        score url for download process
@@ -34,6 +34,12 @@ process scoreUpload {
 
     container "${ params.score_container ?: params.container}:${params.score_container_version ?: params.container_version}"
 
+    if (workflow.containerEngine == "singularity") {
+        containerOptions "--bind \$(pwd):/score-client/logs"
+    } else if (workflow.containerEngine == "docker") {
+        containerOptions "-v \$(pwd):/score-client/logs"
+    }
+
     tag "${analysis_id}"
 
     input:
@@ -46,13 +52,14 @@ process scoreUpload {
 
     script:
         accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`"
+        forceFlag = params.score_force ? "--force" : ""
         """
         export METADATA_URL=${params.song_url}
         export STORAGE_URL=${params.score_url}
         export TRANSPORT_PARALLEL=${params.cpus}
         export TRANSPORT_MEMORY=${params.transport_mem}
         export ACCESSTOKEN=${accessToken}
         
-        score-client upload --manifest ${manifest}
+        score-client upload --manifest ${manifest} ${forceFlag}
         """
 }
diff --git a/[email protected]/local_modules/song-manifest.nf → [email protected]/local_modules/song-manifest.nf b/[email protected]/local_modules/song-manifest.nf → [email protected]/local_modules/song-manifest.nf
@@ -12,7 +12,7 @@ params.first_retry_wait_time = 1  // in seconds
 
 // required params w/ default
 params.container = "ghcr.io/overture-stack/song-client"
-params.container_version = "5.0.2"
+params.container_version = "latest"
 
 // optional if secret mounted from pod else required
 params.api_token = "" // song/score API token for download process
@@ -37,6 +37,12 @@ process songManifest {
     memory "${params.mem} GB"
     tag "${analysis_id}"
 
+    if (workflow.containerEngine == "singularity") {
+        containerOptions "--bind \$(pwd):/song-client/logs"
+    } else if (workflow.containerEngine == "docker") {
+        containerOptions "-v \$(pwd):/song-client/logs"
+    }
+
     input:
         val study_id
         val analysis_id

diff --git a/[email protected]/local_modules/song-publish.nf → [email protected]/local_modules/song-publish.nf b/[email protected]/local_modules/song-publish.nf → [email protected]/local_modules/song-publish.nf
@@ -10,7 +10,7 @@ params.first_retry_wait_time = 1  // in seconds
 
 // required params w/ default
 params.container = "ghcr.io/overture-stack/song-client"
-params.container_version = "5.0.2"
+params.container_version = "latest"
 
 // optional if secret mounted from pod else required
 params.api_token = "" // song/score API token for download process
@@ -33,6 +33,12 @@ process songPublish {
 
     container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}"
 
+    if (workflow.containerEngine == "singularity") {
+        containerOptions "--bind \$(pwd):/song-client/logs"
+    } else if (workflow.containerEngine == "docker") {
+        containerOptions "-v \$(pwd):/song-client/logs"
+    }
+
     tag "${analysis_id}"
 
     input:

diff --git a/[email protected]/local_modules/song-submit.nf → [email protected]/local_modules/song-submit.nf b/[email protected]/local_modules/song-submit.nf → [email protected]/local_modules/song-submit.nf
@@ -10,7 +10,7 @@ params.first_retry_wait_time = 1  // in seconds
 
 // required params w/ default
 params.container = "ghcr.io/overture-stack/song-client"
-params.container_version = "5.0.2"
+params.container_version = "latest"
 
 // optional if secret mounted from pod else required
 params.api_token = "" // song/score API token for download process
@@ -33,6 +33,12 @@ process songSubmit {
 
     container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}"
 
+    if (workflow.containerEngine == "singularity") {
+        containerOptions "--bind \$(pwd):/song-client/logs"
+    } else if (workflow.containerEngine == "docker") {
+        containerOptions "-v \$(pwd):/song-client/logs"
+    }
+
     tag "${study_id}"
     label "songSubmit"
 

diff --git a/...ity-tools/[email protected]/main.nf → ...ity-tools/[email protected]/main.nf b/...ity-tools/[email protected]/main.nf → ...ity-tools/[email protected]/main.nf
@@ -21,7 +21,7 @@
 */
 
 nextflow.enable.dsl = 2
-version = '2.9.0'
+version = '2.9.3'
 
 // universal params go here, change default value as needed
 params.publish_dir = ""  // set to empty string will disable publishDir
@@ -42,15 +42,16 @@ params.song_mem = 1  // GB
 params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
 params.song_api_token = ""
 params.song_container = "ghcr.io/overture-stack/song-client"
-params.song_container_version = "5.0.2"
+params.song_container_version = "latest"
 
 params.score_cpus = 1
 params.score_mem = 1  // GB
 params.score_transport_mem = 1  // GB
 params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
 params.score_api_token = ""
 params.score_container = "ghcr.io/overture-stack/score"
-params.score_container_version = "5.8.1"
+params.score_container_version = "latest"
+params.score_force = false
 
 song_params = [
     *:params,
@@ -71,7 +72,8 @@ score_params = [
     'score_url': params.score_url,
     'score_container': params.score_container,
     'score_container_version': params.score_container_version,
-    'api_token': params.score_api_token ?: params.api_token
+    'api_token': params.score_api_token ?: params.api_token,
+    'score_force': params.score_force
 ]
 
 include { songSubmit as songSub } from './local_modules/song-submit' params(song_params)

diff --git a/...s/[email protected]/nextflow.config → ...s/[email protected]/nextflow.config b/...s/[email protected]/nextflow.config → ...s/[email protected]/nextflow.config
diff --git a/...ty-tools/[email protected]/pkg.json → ...ty-tools/[email protected]/pkg.json b/...ty-tools/[email protected]/pkg.json → ...ty-tools/[email protected]/pkg.json
@@ -1,6 +1,6 @@
 {
     "name": "song-score-upload",
-    "version": "2.9.0",
+    "version": "2.9.3",
     "description": "SONG/SCORE upload",
     "main": "main.nf",
     "deprecated": false,

diff --git a/...ools/[email protected]/wfpr_modules → ...ools/[email protected]/wfpr_modules b/...ools/[email protected]/wfpr_modules → ...ools/[email protected]/wfpr_modules