Skip to content

Commit

Permalink
Merge pull request #9 from ENCODE-DCC/develop_v1
Browse files Browse the repository at this point in the history 
Develop v1
  • Loading branch information
@leepc12
leepc12 authored Jun 4, 2018
2 parents 52a72a6 + d7d8815 commit 808efa3
Show file tree
Hide file tree
Showing 187 changed files with 7,843 additions and 2,689 deletions.
4 changes: 4 additions & 0 deletions .dockerignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
.DS_Store
.editorconfig
.git
.gitignore
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,3 +111,5 @@ cromwell*.jar
#test.wdl
#test_google.wdl
output_*.json
test/*_standalone.wdl
.DS_Store
146 changes: 124 additions & 22 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,144 @@
pipeline {
agent {label 'slave-w-docker-cromwell-60GB-ebs'}

agent none
environment {
QUAY_USER = credentials('quay-robot')
QUAY_PASS = credentials('quay-robot-token')
}
stages {
stage('Unit-tests') {
steps {
echo "Running unit tests.."
}
}
stage('Tag Non-master') {
agent {label 'master-builder'}
when { not {branch 'master'}}
steps {
// the tag gets built here, and can be referenced in the other stages
script {
TAG = sh([script: "echo quay.io/encode-dcc/atac-seq-pipeline:${env.BRANCH_NAME}_${env.BUILD_NUMBER}", returnStdout: true]).trim()
}
echo "On non-master"
}

}

stage('Tag Master') {
agent {label 'master-builder'}
when { branch 'master'}
steps {
// the tag gets built here, and can be referenced in the other stages
script {
TAG = sh([script: "echo quay.io/encode-dcc/atac-seq-pipeline:latest", returnStdout: true]).trim()
}
echo "On non-master"
}
}

stage('Build-nonmaster') {
agent {label 'slave-w-docker-cromwell-60GB-ebs'}
when { not { branch 'master' } }
steps {
slackSend "started job: ${env.JOB_NAME}, build number ${env.BUILD_NUMBER} on branch: ${env.BRANCH_NAME}."
slackSend "The images will be tagged as ${env.BRANCH_NAME}:${env.BUILD_NUMBER}"
echo "${env.BRANCH_NAME}"
echo "Running non-master build steps."
echo "the tag is $TAG"
echo "going to build a docker image now.."
slackSend (color: '#7CFC00', message: "started job: ${env.JOB_NAME}, build number ${env.BUILD_NUMBER} on branch: ${env.BRANCH_NAME}.")
slackSend "The images will be tagged as $TAG"

// pull the cache template image (the image is going to stay pretty much the same so it is no need to be dynamic)
sh "docker pull quay.io/encode-dcc/atac-seq-pipeline:v1"
sh "docker login -u=${QUAY_USER} -p=${QUAY_PASS} quay.io"
sh "docker build --cache-from quay.io/encode-dcc/atac-seq-pipeline:v1 -f docker_image/Dockerfile -t atac-seq-pipeline ."
sh "docker tag atac-seq-pipeline $TAG"
sh "docker push $TAG"
sh "docker logout"
}
}

stage('Build-master') {
agent {label 'slave-w-docker-cromwell-60GB-ebs'}
when { branch 'master'}
steps {
slackSend "started job: ${env.JOB_NAME}, build number ${env.BUILD_NUMBER} on branch: ${env.BRANCH_NAME}."
slackSend "The images will be tagged as ${env.BRANCH_NAME}:${env.BUILD_NUMBER}"
echo "${env.BRANCH_NAME}"
echo "Running master build steps."
echo "going to build a docker image now.."
slackSend (color: '#7CFC00', message: "started job: ${env.JOB_NAME}, build number ${env.BUILD_NUMBER} on branch: ${env.BRANCH_NAME}.")
slackSend "The images will be tagged as quay.io/encode-dcc/atac-seq-pipeline:latest"
// pull the cache template image (the image is going to stay pretty much the same so it is no need to be dynamic)
sh "docker pull quay.io/encode-dcc/atac-seq-pipeline:v1"
sh "docker login -u=${QUAY_USER} -p=${QUAY_PASS} quay.io"
sh "docker build --cache-from quay.io/encode-dcc/atac-seq-pipeline:v1 -f docker_image/Dockerfile -t atac-seq-pipeline ."
sh "docker tag atac-seq-pipeline quay.io/encode-dcc/atac-seq-pipeline:latest"
sh "docker push quay.io/encode-dcc/atac-seq-pipeline:latest"
sh "docker logout"
}
}

stage('Run-Task-Level-Tests-Non-Master'){
agent {label 'slave-w-docker-cromwell-60GB-ebs'}
steps{
sh "cd test/test_task && git clone https://github.com/ENCODE-DCC/atac-seq-pipeline-test-data"
sh """cd test/test_task
./download_hg38_fasta_for_test_ataqc.sh
./test.sh test_bam2ta.wdl test_bam2ta.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_bam2ta.result.json
./test.sh test_bowtie2.wdl test_bowtie2.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_bowtie2.result.json
./test.sh test_filter.wdl test_filter.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_filter.result.json
./test.sh test_idr.wdl test_idr.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_idr.result.json
./test.sh test_macs2.wdl test_macs2.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_macs2.result.json
./test.sh test_overlap.wdl test_overlap.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_overlap.result.json
./test.sh test_pool_ta.wdl test_pool_ta.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_pool_ta.result.json
./test.sh test_reproducibility.wdl test_reproducibility.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_reproducibility.result.json
./test.sh test_spr.wdl test_spr.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_spr.result.json
./test.sh test_trim_adapter.wdl test_trim_adapter.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_trim_adapter.result.json
./test.sh test_xcor.wdl test_xcor.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_xcor.result.json
./test.sh test_ataqc.wdl test_ataqc.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'match_overall']))" < test_ataqc.result.json
"""
}
}

stage('Run-Workflow-Level-Quick-Tests'){
agent {label 'master-builder'}
when {not {branch 'master'}}
steps {
echo "running subsampled/chr19_only/paired_end workflow level tests when there is an event on master branch"
//sh """cd test/test_workflow
// ./test_atac.sh ENCSR356KRQ_subsampled_chr19_only.json $TAG
// python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'outputs'][u'atac.qc_report.qc_json_match']))" < ENCSR356KRQ_subsampled_chr19_only.result.json
// """
}
}

stage('Run-Workflow-Level-Full-Tests'){
agent {label 'slave-w-docker-cromwell-60GB-ebs'}
when { branch 'master'}
steps {
echo "running subsampled/paired_end workflow level tests when there is an event on master branch"
sh """
./test_atac.sh ENCSR356KRQ_subsampled.json $TAG
python -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data[u'outputs'][u'atac.qc_report.qc_json_match']))" < ENCSR356KRQ_subsampled.result.json
"""
}
}
}

post {
success {
echo 'Post build actions that run on success'
slackSend "Job ${env.JOB_NAME}, build number ${env.BUILD_NUMBER} on branch ${env.BRANCH_NAME} finished successfully."
echo "Post build actions that run on success"
slackSend "Job ${env.JOB_NAME}, build number ${env.BUILD_NUMBER} on branch ${env.BRANCH_NAME} finished with"
slackSend (color: '#7cfc00', message: "SUCCESS")
slackSend "For details, visit ${env.BUILD_URL}"
}
failure {
echo 'Post build actions that run on failure'
slackSend "Job ${env.JOB_NAME}, build number ${env.BUILD_NUMBER} on branch ${env.BRANCH_NAME} failed."
}
always {
echo 'Post build actions that run always'
echo "Post build actions that run on failure"
slackSend "Job ${env.JOB_NAME}, build number ${env.BUILD_NUMBER} on branch ${env.BRANCH_NAME} finished with"
slackSend (color: '#FF0000', message: "FAILURE")
slackSend "For details, visit ${env.BUILD_URL}"

}

}
}
Loading

0 comments on commit 808efa3

Please sign in to comment.