Merge branch 'develop'

centaur/src/main/config/cromwell-account.pem.ctmpl

@@ -1 +1 @@
-{{with $environment := env "ENVIRONMENT"}}{{$keyname := printf "secret/dsde/cromwell/%s/cromwell/secrets02" $environment}}{{with vault $keyname}}{{.Data.service_account.private_key}}{{end}}{{end}}
+{{with $environment := env "ENVIRONMENT"}}{{$keyname := printf "secret/dsde/cromwell/%s/cromwell/secrets02" $environment}}{{with secret $keyname}}{{.Data.service_account.private_key}}{{end}}{{end}}

cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala

@@ -104,5 +104,5 @@ case class GcsUriDownloader(gcsUrl: String,
 }
 
 object GcsUriDownloader {
-  private final val RequesterPaysErrorMsg = "Bucket is requester pays bucket but no user project provided."
+  private final val RequesterPaysErrorMsg = "requester pays bucket but no user project"
 }

cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GcsUriDownloaderSpec.scala

@@ -77,7 +77,7 @@ class GcsUriDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
          |
          |if [ "$$RC_GSUTIL" != "0" ]; then
          |  # Check if error is requester pays. If yes, retry gsutil copy using project flag
-         |  if grep -q 'Bucket is requester pays bucket but no user project provided.' gsutil_output.txt; then
+         |  if grep -q 'requester pays bucket but no user project' gsutil_output.txt; then
          |    echo "Received 'Bucket is requester pays' error. Attempting again using Requester Pays billing project"
          |    gsutil -u fake-billing-project cp $gcsUrl $fakeDownloadLocation > gsutil_output.txt 2>&1
          |    RC_GSUTIL=$$?

filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsReader.scala

@@ -96,7 +96,7 @@ case class GcsReader(googleAuthMode: GoogleAuthMode,
         throwable =>
           (requesterPaysProjectIdOption, throwable) match {
             case (Some(requesterPaysProjectId), storageException: StorageException)
-              if storageException.getMessage == "Bucket is requester pays bucket but no user project provided." =>
+              if storageException.getMessage.contains("requester pays bucket but no user project") =>
               IO(
                 storage
                   .get(bucketName, objectName, BlobGetOption.userProject(requesterPaysProjectId))

filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala

@@ -6,7 +6,7 @@ import org.apache.commons.lang3.StringUtils
 
 object RequesterPaysErrors {
   val BucketIsRequesterPaysErrorCode = 400
-  val BucketIsRequesterPaysErrorMessage = "Bucket is requester pays bucket but no user project provided."
+  val BucketIsRequesterPaysErrorMessage = "requester pays bucket but no user project"
   val DoesNotHaveServiceUsePermissionErrorCode = 403
   val DoesNotHaveServiceUsePermissionErrorMessage = "does not have serviceusage.services.use"
 

filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsEnhancedRequestSpec.scala

@@ -17,8 +17,8 @@ class GcsEnhancedRequestSpec extends AnyFlatSpec with CromwellTimeoutSpec with M
   behavior of "GcsEnhancedRequest"
 
   val path = GcsPath(CloudStorageFileSystem.forBucket("bucket").getPath("test"), any[com.google.api.services.storage.Storage], any[com.google.cloud.storage.Storage], anyString)
-  val requesterPaysException = new StorageException(BucketIsRequesterPaysErrorCode, BucketIsRequesterPaysErrorMessage)
-  
+  val requesterPaysException = new StorageException(BucketIsRequesterPaysErrorCode, "Bucket is a requester pays bucket but no user project provided.")
+
   it should "attempt first without project, and not retry if the requests succeeds" in {
     val testFunction = mockFunction[Boolean, String]
     testFunction.expects(false).returns("hello").once()
@@ -67,7 +67,7 @@ class GcsEnhancedRequestSpec extends AnyFlatSpec with CromwellTimeoutSpec with M
     val builder = new HttpResponseException.Builder(404, "NotFound", new HttpHeaders)
     val error = new GoogleJsonError()
     error.setCode(404)
-    
+
     // Throw an unrelated exception, should only be called once
     testFunction.expects(false).throws(new GoogleJsonResponseException(builder, error)).once()
     a[FileNotFoundException] should be thrownBy GcsEnhancedRequest.recoverFromProjectNotProvided(path, testFunction).unsafeRunSync()

project/Version.scala

@@ -5,7 +5,7 @@ import sbt._
 
 object Version {
   // Upcoming release, or current if we're on a master / hotfix branch
-  val cromwellVersion = "76"
+  val cromwellVersion = "77"
 
   /**
     * Returns true if this project should be considered a snapshot.

runConfigurations/Repo template_ CromIAM.run.xml

@@ -0,0 +1,10 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Repo template: CromIAM" type="Application" factoryName="Application">
+    <option name="MAIN_CLASS_NAME" value="cromiam.Main" />
+    <module name="cromiam" />
+    <option name="PROGRAM_PARAMETERS" value="-D sam.interface=https://sam.dsde-dev.broadinstitute.org" />
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>

src/ci/resources/artifactory_credentials.properties.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellArtifactory := vault (printf "secret/dsde/cromwell/common/cromwell-artifactory")}}
+{{with $cromwellArtifactory := secret (printf "secret/dsde/cromwell/common/cromwell-artifactory")}}
 realm=Artifactory Realm
 host=broadinstitute.jfrog.io
 user={{$cromwellArtifactory.Data.username}}

src/ci/resources/aws_credentials.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellAws := vault (printf "secret/dsde/cromwell/common/cromwell-aws")}}
+{{with $cromwellAws := secret (printf "secret/dsde/cromwell/common/cromwell-aws")}}
 [default]
 aws_access_key_id = {{$cromwellAws.Data.access_key}}
 aws_secret_access_key = {{$cromwellAws.Data.secret_key}}

src/ci/resources/bcs_application.conf.ctmpl

@@ -1,7 +1,7 @@
 include required(classpath("application.conf"))
 include "build_application.inc.conf"
 
-{{with $cromwellBcs := vault (printf "secret/dsde/cromwell/common/cromwell-bcs")}}
+{{with $cromwellBcs := secret (printf "secret/dsde/cromwell/common/cromwell-bcs")}}
 backend {
   default = "BCS"
 

src/ci/resources/bcs_login.inc.sh.ctmpl

@@ -8,7 +8,7 @@ set -o errexit -o nounset -o pipefail
 local bcs_access_id
 local bcs_access_key
 
-{{with $cromwellBcs := vault (printf "secret/dsde/cromwell/common/cromwell-bcs")}}
+{{with $cromwellBcs := secret (printf "secret/dsde/cromwell/common/cromwell-bcs")}}
 bcs_access_id="{{$cromwellBcs.Data.access_id}}"
 bcs_access_key="{{$cromwellBcs.Data.access_key}}"
 {{end}}

src/ci/resources/cromwell-centaur-google-compute-service-account.json.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-centaur-google-compute-service-account.json")}}
+{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-centaur-google-compute-service-account.json")}}
 
 {{$cromwellServiceAccount.Data | toJSONPretty}}
 

src/ci/resources/cromwell-centaur-requester-pays-service-account.json.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-centaur-requester-pays-service-account.json")}}
+{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-centaur-requester-pays-service-account.json")}}
 
 {{$cromwellServiceAccount.Data | toJSONPretty}}
 

src/ci/resources/cromwell-centaur-service-account.json.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-centaur-service-account.json")}}
+{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-centaur-service-account.json")}}
 
 {{$cromwellServiceAccount.Data | toJSONPretty}}
 

src/ci/resources/cromwell-perf-service-account.json.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellPerfServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-perf-service-account.json")}}
+{{with $cromwellPerfServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-perf-service-account.json")}}
 
 {{$cromwellPerfServiceAccount.Data | toJSONPretty}}
 

src/ci/resources/cromwell-service-account.json.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
+{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
 
 {{$cromwellServiceAccount.Data | toJSONPretty}}
 

src/ci/resources/dockerhub_provider_config_v1.inc.conf.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellDockerhub := vault (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
+{{with $cromwellDockerhub := secret (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
 dockerhub {
   token = "{{$cromwellDockerhub.Data.token}}"
 }

src/ci/resources/dockerhub_provider_config_v2.inc.conf.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellDockerhub := vault (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
+{{with $cromwellDockerhub := secret (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
 dockerhub {
   token = "{{$cromwellDockerhub.Data.token}}"
   key-name = "{{$cromwellDockerhub.Data.key_name}}"

src/ci/resources/dockerhub_provider_config_v2_usa.inc.conf.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellDockerhub := vault (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
+{{with $cromwellDockerhub := secret (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
 dockerhub {
   token = "{{$cromwellDockerhub.Data.token}}"
   key-name = "{{$cromwellDockerhub.Data.key_name}}"

src/ci/resources/ftp_centaur_cwl_runner.conf.ctmpl

@@ -1,7 +1,7 @@
 include required(classpath("application.conf"))
 include "centaur_cwl_runner_application.conf"
 
-{{with $cromwellFtp := vault (printf "secret/dsde/cromwell/common/cromwell-ftp")}}
+{{with $cromwellFtp := secret (printf "secret/dsde/cromwell/common/cromwell-ftp")}}
 centaur {
   cwl-runner {
     ftp {

src/ci/resources/github_private_deploy_key.ctmpl

@@ -1,3 +1,3 @@
-{{with $cromwellGithub := vault (printf "secret/dsde/cromwell/common/cromwell-github")}}
+{{with $cromwellGithub := secret (printf "secret/dsde/cromwell/common/cromwell-github")}}
 {{$cromwellGithub.Data.private_deploy_key}}
 {{end}}

src/ci/resources/papi_v2_gcsa.options.json.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
+{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
 {
    "google_compute_service_account": "{{$cromwellServiceAccount.Data.client_email}}",
    "google_project": "{{$cromwellServiceAccount.Data.project_id }}",

src/ci/resources/papi_v2_usa.options.json.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
+{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
 {
     "google_compute_service_account": "{{$cromwellServiceAccount.Data.client_email}}",
     "user_service_account_json": "{{$cromwellServiceAccount.Data | toJSON | replaceAll "\\" "\\\\" | replaceAll "\"" "\\\"" }}",

src/ci/resources/papi_v2_usa_preresolve.options.json.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
+{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
 {
     "google_compute_service_account": "{{$cromwellServiceAccount.Data.client_email}}",
     "user_service_account_json": "{{$cromwellServiceAccount.Data | toJSON | replaceAll "\\" "\\\\" | replaceAll "\"" "\\\"" }}",

src/ci/resources/private_docker_papi_v2_usa.options.ctmpl

@@ -1,5 +1,5 @@
-{{with $cromwellDockerhub := vault (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
-{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
+{{with $cromwellDockerhub := secret (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
+{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
 {
    "read_from_cache": false,
    "user_service_account_json": "{{$cromwellServiceAccount.Data | toJSON | replaceAll "\\" "\\\\" | replaceAll "\"" "\\\"" }}",

src/ci/resources/tesk_application.conf.ctmpl

@@ -2,7 +2,7 @@ include required(classpath("application.conf"))
 include "build_application.inc.conf"
 include "tesk_application_ftp.conf"
 
-{{with $cromwellTesk := vault (printf "secret/dsde/cromwell/common/cromwell-tesk")}}
+{{with $cromwellTesk := secret (printf "secret/dsde/cromwell/common/cromwell-tesk")}}
 backend {
   default = "TES"
   providers {

src/ci/resources/tesk_application_ftp.conf.ctmpl

@@ -1,4 +1,4 @@
-{{with $cromwellFtp := vault (printf "secret/dsde/cromwell/common/cromwell-ftp")}}
+{{with $cromwellFtp := secret (printf "secret/dsde/cromwell/common/cromwell-ftp")}}
 filesystems {
    ftp.global.config.max-connection-per-server-per-user = 5
 }

supportedBackends/google/pipelines/v2alpha1/src/test/scala/cromwell/backend/google/pipelines/v2alpha1/api/ActionCommandsSpec.scala

@@ -29,7 +29,7 @@ class ActionCommandsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match
                          |  cat gsutil_output.txt
                          |
                          |  # Check if it matches the BucketIsRequesterPaysErrorMessage
-                         |  if grep -q "Bucket is requester pays bucket but no user project provided." gsutil_output.txt; then
+                         |  if grep -q "requester pays bucket but no user project" gsutil_output.txt; then
                          |    printf '%s %s\n' "$(date -u '+%Y/%m/%d %H:%M:%S')" Retrying\ with\ user\ project
                          |    flag is -u my-project
                          |  else

supportedBackends/google/pipelines/v2beta/src/main/resources/gcs_transfer.sh

@@ -126,7 +126,7 @@ private::determine_requester_pays() {
       USE_REQUESTER_PAYS=${use_requester_pays}
       break
     elif [[ "$use_requester_pays" = "false" ]]; then
-      if grep -q "Bucket is requester pays bucket but no user project provided." ${gsutil_log}; then
+      if grep -q "requester pays bucket but no user project" ${gsutil_log}; then
         use_requester_pays=true
         command="$REQUESTER_PAYS_COMMAND"
       else
@@ -332,7 +332,7 @@ delocalize() {
 
         # If the requester pays status of the GCS bucket is not certain look for requester pays errors.
         if [[ ${rp_status_certain} = false ]]; then
-          if grep -q "Bucket is requester pays bucket but no user project provided." "${gsutil_log}"; then
+          if grep -q "requester pays bucket but no user project" "${gsutil_log}"; then
             timestamped_message "Retrying with user project"
             use_requester_pays=true
             # Do not increment the attempt number, a requester pays failure does not count against retries.

supportedBackends/google/pipelines/v2beta/src/test/scala/cromwell/backend/google/pipelines/v2beta/api/ActionCommandsSpec.scala

@@ -29,7 +29,7 @@ class ActionCommandsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match
                          |  cat gsutil_output.txt
                          |
                          |  # Check if it matches the BucketIsRequesterPaysErrorMessage
-                         |  if grep -q "Bucket is requester pays bucket but no user project provided." gsutil_output.txt; then
+                         |  if grep -q "requester pays bucket but no user project" gsutil_output.txt; then
                          |    printf '%s %s\n' "$(date -u '+%Y/%m/%d %H:%M:%S')" Retrying\ with\ user\ project
                          |    flag is -u my-project
                          |  else