Merge branch 'develop'

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Cromwell Change Log
 
+## 64 Release Notes
+
+### Intel Cascade Lake support on PAPI v2
+
+On the PAPI v2 backends "Intel Cascade Lake" is now supported as a CPU platform. More details can be found
+[here](https://cromwell.readthedocs.io/en/develop/RuntimeAttributes/#cpuplatform).
+
 ## 63 Release Notes
 
 ### Removed refresh token authentication mode

build.sbt

@@ -208,13 +208,6 @@ lazy val googlePipelinesCommon = (project in backendRoot / "google" / "pipelines
   .dependsOn(services % "test->test")
   .dependsOn(common % "test->test")
 
-lazy val googlePipelinesV1Alpha2 = (project in backendRoot / "google" / "pipelines" / "v1alpha2")
-  .withLibrarySettings("cromwell-pipelines-v1-backend")
-  .dependsOn(googlePipelinesCommon)
-  .dependsOn(googlePipelinesCommon % "test->test")
-  .dependsOn(core % "test->test")
-  .dependsOn(common % "test->test")
-
 lazy val googlePipelinesV2Alpha1 = (project in backendRoot / "google" / "pipelines" / "v2alpha1")
   .withLibrarySettings("cromwell-pipelines-v2-alpha1-backend")
   .dependsOn(googlePipelinesCommon)
@@ -229,12 +222,6 @@ lazy val googlePipelinesV2Beta = (project in backendRoot / "google" / "pipelines
   .dependsOn(core % "test->test")
   .dependsOn(common % "test->test")
 
-// Legacy, inherits all its code from googlePipelinesV1Alpha2
-lazy val jesBackend = (project in backendRoot / "jes")
-  .withLibrarySettings("cromwell-jes-backend")
-  .dependsOn(googlePipelinesV1Alpha2)
-  .dependsOn(common % "test->test")
-
 lazy val awsBackend = (project in backendRoot / "aws")
   .withLibrarySettings("cromwell-aws-backend")
   .dependsOn(backend)
@@ -404,10 +391,8 @@ lazy val `cromwell-drs-localizer` = project
 lazy val server = project
   .withExecutableSettings("cromwell", serverDependencies)
   .dependsOn(engine)
-  .dependsOn(googlePipelinesV1Alpha2)
   .dependsOn(googlePipelinesV2Alpha1)
   .dependsOn(googlePipelinesV2Beta)
-  .dependsOn(jesBackend)
   .dependsOn(bcsBackend)
   .dependsOn(awsBackend)
   .dependsOn(tesBackend)
@@ -448,11 +433,9 @@ lazy val root = (project in file("."))
   .aggregate(ftpFileSystem)
   .aggregate(gcsFileSystem)
   .aggregate(googlePipelinesCommon)
-  .aggregate(googlePipelinesV1Alpha2)
   .aggregate(googlePipelinesV2Alpha1)
   .aggregate(googlePipelinesV2Beta)
   .aggregate(httpFileSystem)
-  .aggregate(jesBackend)
   .aggregate(languageFactoryCore)
   .aggregate(ossFileSystem)
   .aggregate(perf)

centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala

@@ -6,6 +6,7 @@ import cats.effect.IO
 import cats.instances.list._
 import cats.syntax.flatMap._
 import cats.syntax.traverse._
+import centaur.callcaching.CromwellDatabaseCallCaching
 import centaur.reporting.{ErrorReporters, SuccessReporters, TestEnvironment}
 import centaur.test.CentaurTestException
 import centaur.test.standard.CentaurTestCase
@@ -64,11 +65,7 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw
     // Make tags, but enforce lowercase:
     val tags = (testCase.testOptions.tags :+ testCase.workflow.testName :+ testCase.testFormat.name) map { x => Tag(x.toLowerCase) }
     val isIgnored = testCase.isIgnored(cromwellBackends)
-    val retries =
-      // in this case retrying may end up to be waste of time if some tasks have been cached on previous test attempts
-      if (testCase.reliesOnCallCachingMetadataVerification) 0
-      else if (testCase.workflow.retryTestFailures) ErrorReporters.retryAttempts
-      else 0
+    val retries = ErrorReporters.retryAttempts
 
     runOrDont(nameTest, tags, isIgnored, retries, runTestAndDeleteZippedImports())
   }
@@ -116,9 +113,9 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw
           workflowContent = Option(upgradeResult.stdout.get), // this '.get' catches an error if upgrade fails
           zippedImports = Option(upgradedImportsDir.zip()))))(cromwellTracker) // An empty zip appears to be completely harmless, so no special handling
 
-    rootWorkflowFile.delete(true)
-    upgradedImportsDir.delete(true)
-    workingDir.delete(true)
+    rootWorkflowFile.delete(swallowIOExceptions = true)
+    upgradedImportsDir.delete(swallowIOExceptions = true)
+    workingDir.delete(swallowIOExceptions = true)
 
     newCase
   }
@@ -178,6 +175,10 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw
       for {
         _ <- ErrorReporters.logFailure(testEnvironment, centaurTestException)
         r <- if (attempt < retries) {
+          centaurTestException
+            .workflowIdOption
+            .map(CromwellDatabaseCallCaching.clearCachedResults)
+            .getOrElse(IO.unit) *>
           tryTryAgain(testName, runTest, retries, attempt + 1)
         } else {
           IO.raiseError(centaurTestException)

centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala

@@ -19,7 +19,7 @@ abstract class AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBacken
 
   def this() = this(CentaurTestSuite.cromwellBackends)
 
-  private val cromwellDatabase = CromwellDatabase.fromConfig(CentaurConfig.conf)
+  private val cromwellDatabase = CromwellDatabase.instance
   private val engineSlickDatabaseOption = cromwellDatabase.engineDatabase.cast[EngineSlickDatabase]
   private val metadataSlickDatabaseOption = cromwellDatabase.metadataDatabase.cast[MetadataSlickDatabase]
   import TestContext._

centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala

@@ -0,0 +1,17 @@
+package centaur.callcaching
+
+import cats.effect.IO
+import centaur.CromwellDatabase
+import com.typesafe.scalalogging.StrictLogging
+
+import scala.concurrent.ExecutionContext
+
+object CromwellDatabaseCallCaching extends StrictLogging {
+  import centaur.TestContext._
+
+  private val cromwellDatabase = CromwellDatabase.instance
+
+  def clearCachedResults(workflowId: String)(implicit executionContext: ExecutionContext): IO[Unit] = {
+    IO.fromFuture(IO(cromwellDatabase.engineDatabase.invalidateCallCacheEntryIdsForWorkflowId(workflowId)))
+  }
+}

centaur/src/it/scala/centaur/reporting/ErrorReporters.scala

@@ -18,8 +18,7 @@ class ErrorReporters(rootConfig: Config) {
 
   private val errorReporterConfig: Config = rootConfig.getOrElse("error-reporter", ConfigFactory.empty)
 
-  // Parameters to CromwellDatabase() should be lazy and are only initialized when needed.
-  private val cromwellDatabase = new ErrorReporterCromwellDatabase(CromwellDatabase.fromConfig(CentaurConfig.conf))
+  private val errorReporterCromwellDatabase = new ErrorReporterCromwellDatabase(CromwellDatabase.instance)
 
   private val errorReporterNames: List[String] = {
     val providersConfig = errorReporterConfig.getOrElse("providers", ConfigFactory.empty)
@@ -67,7 +66,7 @@ class ErrorReporters(rootConfig: Config) {
       val clazz = errorReporterConfig.getString(s"providers.$errorReporterName.class")
       val reporterConfig = errorReporterConfig.getOrElse(s"providers.$errorReporterName.config", ConfigFactory.empty)
       val constructor = Class.forName(clazz).getConstructor(classOf[ErrorReporterParams])
-      val params = ErrorReporterParams(errorReporterName, rootConfig, reporterConfig, cromwellDatabase)
+      val params = ErrorReporterParams(errorReporterName, rootConfig, reporterConfig, errorReporterCromwellDatabase)
       constructor.newInstance(params).asInstanceOf[ErrorReporter]
     }
   }
@@ -76,7 +75,7 @@ class ErrorReporters(rootConfig: Config) {
 object ErrorReporters extends StrictLogging {
   private val ciEnvironment = CiEnvironment()
   private[reporting] val errorReporters = new ErrorReporters(CentaurConfig.conf)
-  val retryAttempts = errorReporters.retryAttempts
+  val retryAttempts: Int = errorReporters.retryAttempts
 
   errorReporters.errorReporters foreach { errorReporter =>
     logger.info("Error reporter loaded: {} to {}", errorReporter.params.name, errorReporter.destination)

centaur/src/main/resources/standardTestCases/curl/curl.wdl

@@ -8,15 +8,15 @@ task url_grab {
         File progressBar = stderr()
     }
     runtime {
-        docker: "tutum/curl:latest"
+        docker: "google/cloud-sdk"
         failOnStderr: false
     }
 }
 
 task news_reader {
     File news
 
-    command { 
+    command {
       wc -l < ${news}
     }
     output { Int news_size = read_int(stdout()) }

centaur/src/main/resources/standardTestCases/fast_fail_noAddress.test

@@ -0,0 +1,15 @@
+# NB: To request this test by name, make it lowercase, eg sbt "centaur/it:testOnly * -- -n fast_fail_noaddress"
+name: fast_fail_noAddress
+backends: [Papi,Papiv2]
+backendsMode: any
+testFormat: workflowfailure
+
+files {
+  workflow: fast_fail_noAddress/fast_fail_noAddress.wdl
+}
+
+metadata {
+  workflowName: fast_fail_noAddress
+  "failures.0.causedBy.0.message": "Task fast_fail_noAddress.task_with_noAddress:NA:1 failed. The job was stopped before the command finished. PAPI error code 9. The worker was unable to check in, possibly due to a misconfigured network"
+  status: Failed
+}

centaur/src/main/resources/standardTestCases/ignore_noAddress/ignore_noAddress.wdl → centaur/src/main/resources/standardTestCases/fast_fail_noAddress/fast_fail_noAddress.wdl

@@ -6,12 +6,11 @@ task task_with_noAddress {
 
     runtime {
       docker: "ubuntu:latest"
-      # NB: This is ignored because of the config in the centaur configuration.
-      # If we didn't have that, this task would run forever - or at least until PAPI times it out.
+      # Our network is not configured specially, so this should cause this task to fail almost immediately.
       noAddress: true
     }
 }
 
-workflow ignore_noAddress {
+workflow fast_fail_noAddress {
     call task_with_noAddress
 }

centaur/src/main/resources/standardTestCases/papi_cpu_platform.test

@@ -8,6 +8,7 @@ files {
 
 metadata {
   status: Succeeded
+  "outputs.cpus.cascadeLake.cpuPlatform": "Intel Cascade Lake"
   "outputs.cpus.broadwell.cpuPlatform": "Intel Broadwell"
   "outputs.cpus.haswell.cpuPlatform": "Intel Haswell"
 }

centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl

@@ -26,4 +26,5 @@ task cpu_platform {
 workflow cpus {
    call cpu_platform as haswell { input: cpu_platform = "Intel Haswell" }
    call cpu_platform as broadwell { input: cpu_platform = "Intel Broadwell" }
+   call cpu_platform as cascadeLake { input: cpu_platform = "Intel Cascade Lake" }
 }

centaur/src/main/resources/standardTestCases/papi_delocalization_required_files.test

@@ -0,0 +1,15 @@
+name: papi_delocalization_required_files
+testFormat: workflowfailure
+backends: [Papiv2]
+
+files {
+  workflow: papi_delocalization_required_files/papi_delocalization_required_files.wdl
+}
+
+metadata {
+  workflowName: required_files
+  "calls.required_files.check_it.executionStatus": "Done"
+  "calls.required_files.do_it.executionStatus": "Failed"
+  "calls.required_files.do_it.retryableFailure": "false"
+  "calls.required_files.do_it.failures.0.message": "Task required_files.do_it:NA:1 failed. Job exited without an error, exit code 0. PAPI error code 9. Please check the log file for more details: gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci/required_files/<<UUID>>/call-do_it/do_it.log."
+}

centaur/src/main/resources/standardTestCases/papi_delocalization_required_files/papi_delocalization_required_files.wdl

@@ -0,0 +1,40 @@
+version 1.0
+
+task do_it {
+    input {
+        String sync
+    }
+    command {
+        echo "foo" > file.txt
+    }
+    output {
+        # Intentionally not the right file name. Cromwell's file evaluation should make 'oops.txt'
+        # required so this job should fail.
+        String value = read_string("oops.txt")
+    }
+    runtime {
+        docker: "ubuntu:latest"
+    }
+}
+
+task check_it {
+    command <<<
+      # Create the expected output.
+      echo "foo" > file.txt
+
+      # Make sure the expected output was specified as "required" in the delocalization script.
+      set -euo pipefail
+      grep -A 1 '"/cromwell_root/file.txt"' /cromwell_root/gcs_delocalization.sh  | tail -1 | grep '"required"'
+    >>>
+    output {
+        String value = read_string("file.txt")
+    }
+    runtime {
+        docker: "ubuntu:latest"
+    }
+}
+
+workflow required_files {
+    call check_it
+    call do_it { input: sync = check_it.value }
+}

centaur/src/main/scala/centaur/CromwellDatabase.scala

@@ -10,10 +10,13 @@ trait CromwellDatabase {
 }
 
 object CromwellDatabase {
+
+  lazy val instance: CromwellDatabase = fromConfig(CentaurConfig.conf)
+
   /**
     * Wraps connections to a cromwell database. The database connections are not initialized until first use.
     */
-  def fromConfig(rootConfig: Config): CromwellDatabase = {
+  private def fromConfig(rootConfig: Config): CromwellDatabase = {
     lazy val cromwellConfig = rootConfig.getConfig("cromwell")
     new CromwellDatabase {
       override lazy val engineDatabase: EngineSqlDatabase = EngineSlickDatabase.fromParentConfig(cromwellConfig)

centaur/src/main/scala/centaur/test/Test.scala

@@ -10,7 +10,7 @@ import centaur.test.metadata.WorkflowFlatMetadata
 import centaur.test.metadata.WorkflowFlatMetadata._
 import centaur.test.submit.SubmitHttpResponse
 import centaur.test.workflow.Workflow
-import com.google.api.services.genomics.{Genomics, GenomicsScopes}
+import com.google.api.services.genomics.v2alpha1.{Genomics, GenomicsScopes}
 import com.google.api.services.storage.StorageScopes
 import com.google.auth.Credentials
 import com.google.auth.http.HttpCredentialsAdapter
@@ -384,7 +384,7 @@ object Operations extends StrictLogging {
     new Test[Unit] {
       def checkPAPIAborted(): IO[Unit] = {
         for {
-          operation <- IO { genomics.operations().get(jobId).execute() }
+          operation <- IO { genomics.projects().operations().get(jobId).execute() }
           done = operation.getDone
           operationError = Option(operation.getError)
           aborted = operationError.exists(_.getCode == 1) && operationError.exists(_.getMessage.startsWith("Operation canceled"))

centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala

@@ -52,14 +52,6 @@ case class CentaurTestCase(workflow: Workflow,
   }
 
   def containsTag(tag: String): Boolean = testOptions.tags.contains(tag)
-
-  def reliesOnCallCachingMetadataVerification: Boolean = {
-    val callCachingMetadataKey = "callCaching"
-    workflow.metadata match {
-      case Some(flatMetadata) => flatMetadata.value.keySet.exists(_.contains(callCachingMetadataKey))
-      case None => false
-    }
-  }
 }
 
 object CentaurTestCase {

cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala

@@ -47,11 +47,11 @@ class DrsCloudNioFileProvider(drsPathResolver: EngineDrsPathResolver,
     throw new UnsupportedOperationException("DRS currently doesn't support delete.")
 
   override def read(drsPath: String, unused: String, offset: Long): ReadableByteChannel = {
-    val fields = NonEmptyList.of(MarthaField.GsUri, MarthaField.GoogleServiceAccount)
+    val fields = NonEmptyList.of(MarthaField.GsUri, MarthaField.GoogleServiceAccount, MarthaField.AccessUrl)
 
     val byteChannelIO = for {
       marthaResponse <- drsPathResolver.resolveDrsThroughMartha(drsPath, fields)
-      byteChannel <- drsReadInterpreter(marthaResponse.gsUri, marthaResponse.googleServiceAccount)
+      byteChannel <- drsReadInterpreter(drsPathResolver, marthaResponse)
     } yield byteChannel
 
     byteChannelIO.handleErrorWith {
@@ -68,15 +68,15 @@ class DrsCloudNioFileProvider(drsPathResolver: EngineDrsPathResolver,
     val fileAttributesIO = for {
       marthaResponse <- drsPathResolver.resolveDrsThroughMartha(drsPath, fields)
       sizeOption = marthaResponse.size
-      hashoption = getPreferredHash(marthaResponse.hashes)
+      hashOption = getPreferredHash(marthaResponse.hashes)
       timeCreatedOption <- convertToFileTime(drsPath, MarthaField.TimeCreated, marthaResponse.timeCreated)
       timeUpdatedOption <- convertToFileTime(drsPath, MarthaField.TimeUpdated, marthaResponse.timeUpdated)
-    } yield new DrsCloudNioRegularFileAttributes(drsPath, sizeOption, hashoption, timeCreatedOption, timeUpdatedOption)
+    } yield new DrsCloudNioRegularFileAttributes(drsPath, sizeOption, hashOption, timeCreatedOption, timeUpdatedOption)
 
     Option(fileAttributesIO.unsafeRunSync())
   }
 }
 
 object DrsCloudNioFileProvider {
-  type DrsReadInterpreter = (Option[String], Option[SADataObject]) => IO[ReadableByteChannel]
+  type DrsReadInterpreter = (DrsPathResolver, MarthaResponse) => IO[ReadableByteChannel]
 }