From 12c2be576a664891f1eedc77eb3f379eec748e1f Mon Sep 17 00:00:00 2001
From: Brian Reilly <breilly@broadinstitute.org>
Date: Tue, 22 Feb 2022 23:17:15 +0000
Subject: [PATCH 01/19] Update cromwell version from 77 to 78

---
 project/Version.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/Version.scala b/project/Version.scala
index 1d03339842f..fb665bce138 100644
--- a/project/Version.scala
+++ b/project/Version.scala
@@ -5,7 +5,7 @@ import sbt._
 
 object Version {
   // Upcoming release, or current if we're on a master / hotfix branch
-  val cromwellVersion = "77"
+  val cromwellVersion = "78"
 
   /**
     * Returns true if this project should be considered a snapshot.

From 24b27a2ff443ad76f760f5293461a3561098f17a Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@users.noreply.github.com>
Date: Wed, 23 Feb 2022 19:27:30 -0500
Subject: [PATCH 02/19] Cromwell Swagger UI Restructuring [CROM-6862] (#6682)

---
 .../cromwell/webservice/SwaggerService.scala  |   4 -
 .../webservice/SwaggerUiHttpService.scala     | 126 ++++++++----------
 .../webservice/SwaggerServiceSpec.scala       |  13 --
 .../webservice/SwaggerUiHttpServiceSpec.scala | 113 +++-------------
 4 files changed, 74 insertions(+), 182 deletions(-)

diff --git a/engine/src/main/scala/cromwell/webservice/SwaggerService.scala b/engine/src/main/scala/cromwell/webservice/SwaggerService.scala
index 0285b3d83f7..182da3ac36c 100644
--- a/engine/src/main/scala/cromwell/webservice/SwaggerService.scala
+++ b/engine/src/main/scala/cromwell/webservice/SwaggerService.scala
@@ -1,9 +1,5 @@
 package cromwell.webservice
 
-import cromwell.webservice.routes.CromwellApiService
-
 trait SwaggerService extends SwaggerUiResourceHttpService {
   override def swaggerServiceName = "cromwell"
-
-  override def swaggerUiVersion = CromwellApiService.swaggerUiVersion
 }
diff --git a/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala b/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala
index f366808e7be..d63ab3bcb51 100644
--- a/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala
+++ b/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala
@@ -1,51 +1,38 @@
 package cromwell.webservice
 
 import akka.http.scaladsl.model.{HttpResponse, StatusCodes}
+import akka.http.scaladsl.server
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server.Route
 import akka.stream.scaladsl.Flow
 import akka.util.ByteString
+import cromwell.webservice.routes.CromwellApiService
 
 /**
- * Serves up the swagger UI from org.webjars/swagger-ui.
+ * Serves up the swagger UI from org.webjars/swagger-ui, lightly editing the index.html.
  */
 trait SwaggerUiHttpService {
-  /**
-   * @return The version of the org.webjars/swagger-ui artifact. For example "2.1.1".
-   */
-  def swaggerUiVersion: String
-
-  /**
-   * Informs the swagger UI of the base of the application url, as hosted on the server.
-   * If your entire app is served under "http://myserver/myapp", then the base URL is "/myapp".
-   * If the app is served at the root of the application, leave this value as the empty string.
-   *
-   * @return The base URL used by the application, or the empty string if there is no base URL. For example "/myapp".
-   */
-  def swaggerUiBaseUrl: String = ""
-
-  /**
-   * @return The path to the swagger UI html documents. For example "swagger"
-   */
-  def swaggerUiPath: String = "swagger"
 
-  /**
-   * The path to the actual swagger documentation in either yaml or json, to be rendered by the swagger UI html.
-   *
-   * @return The path to the api documentation to render in the swagger UI.
-   *         For example "api-docs" or "swagger/common.yaml".
-   */
-  def swaggerUiDocsPath: String = "api-docs"
-
-  /**
-   * @return When true, if someone requests / (or /baseUrl if setup), redirect to the swagger UI.
-   */
-  def swaggerUiFromRoot: Boolean = true
-
-  private def routeFromRoot: Route = get {
-    pathEndOrSingleSlash {
-      // Redirect / to the swagger UI
-      redirect(s"$swaggerUiBaseUrl/$swaggerUiPath", StatusCodes.TemporaryRedirect)
+  private lazy val resourceDirectory = s"META-INF/resources/webjars/swagger-ui/${CromwellApiService.swaggerUiVersion}"
+
+  private val serveIndex: server.Route = {
+    val swaggerOptions =
+      """
+        |        validatorUrl: null,
+        |        apisSorter: "alpha",
+        |        operationsSorter: "alpha"
+      """.stripMargin
+
+    mapResponseEntity { entityFromJar =>
+      entityFromJar.transformDataBytes(Flow.fromFunction[ByteString, ByteString] { original: ByteString =>
+        ByteString(
+          original.utf8String
+            .replace("""url: "https://petstore.swagger.io/v2/swagger.json"""", "url: 'cromwell.yaml'")
+            .replace("""layout: "StandaloneLayout"""", s"""layout: "StandaloneLayout", $swaggerOptions""")
+        )
+      })
+    } {
+      getFromResource(s"$resourceDirectory/index.html")
     }
   }
 
@@ -55,21 +42,32 @@ trait SwaggerUiHttpService {
    * @return Route serving the swagger UI.
    */
   final def swaggerUiRoute: Route = {
-    val route = get {
-      pathPrefix(separateOnSlashes(swaggerUiPath)) {
-        // when the user hits the doc url, redirect to the index.html with api docs specified on the url
-        pathEndOrSingleSlash {
-          redirect(
-            s"$swaggerUiBaseUrl/$swaggerUiPath/index.html?url=$swaggerUiBaseUrl/$swaggerUiDocsPath",
-            StatusCodes.TemporaryRedirect)
-        } ~ getFromResourceDirectory(s"META-INF/resources/webjars/swagger-ui/$swaggerUiVersion")
+    pathEndOrSingleSlash {
+      get {
+        serveIndex
+      }
+    } ~
+      // We have to be explicit about the paths here since we're matching at the root URL and we don't
+      // want to catch all paths lest we circumvent Spray's not-found and method-not-allowed error
+      // messages.
+      (pathPrefixTest("swagger-ui") | pathPrefixTest("oauth2") | pathSuffixTest("js")
+        | pathSuffixTest("css") | pathPrefixTest("favicon")) {
+        get {
+          getFromResourceDirectory(resourceDirectory)
+        }
+      } ~
+      // Redirect legacy `/swagger` or `/swagger/index.html?url=/swagger/cromwell.yaml#fragment` requests to the root
+      // URL. The latter form is (somewhat magically) well-behaved in throwing away the `url` query parameter that was
+      // the subject of the CVE linked below while preserving any fragment identifiers to scroll to the right spot in
+      // the Swagger UI.
+      // https://github.com/swagger-api/swagger-ui/security/advisories/GHSA-qrmm-w75w-3wpx
+      (path("swagger" / "index.html") | path ("swagger")) {
+        get {
+          redirect("/", StatusCodes.MovedPermanently)
+        }
       }
-    }
-    if (swaggerUiFromRoot) route ~ routeFromRoot else route
   }
-
 }
-
 /**
  * An extension of HttpService to serve up a resource containing the swagger api as yaml or json. The resource
  * directory and path on the classpath must match the path for route. The resource can be any file type supported by the
@@ -84,7 +82,7 @@ trait SwaggerResourceHttpService {
   /**
    * @return The directory for the resource under the classpath, and in the url
    */
-  def swaggerDirectory: String = "swagger"
+  private val swaggerDirectory: String = "swagger"
 
   /**
    * @return Name of the service, used to map the documentation resource at "/uiPath/serviceName.resourceType".
@@ -96,29 +94,17 @@ trait SwaggerResourceHttpService {
    */
   def swaggerResourceType: String = "yaml"
 
-  /**
-   * Swagger UI sends HTTP OPTIONS before ALL requests, and expects a status 200 / OK. When true (the default) the
-   * swaggerResourceRoute will return 200 / OK for requests for OPTIONS.
-   *
-   * See also:
-   * - https://github.com/swagger-api/swagger-ui/issues/1209
-   * - https://github.com/swagger-api/swagger-ui/issues/161
-   * - https://groups.google.com/forum/#!topic/swagger-swaggersocket/S6_I6FBjdZ8
-   *
-   * @return True if status code 200 should be returned for HTTP OPTIONS requests for the swagger resource.
-   */
-  def swaggerAllOptionsOk: Boolean = true
-
   /**
    * @return The path to the swagger docs.
    */
-  protected def swaggerDocsPath = s"$swaggerDirectory/$swaggerServiceName.$swaggerResourceType"
+  private lazy val swaggerDocsPath = s"$swaggerDirectory/$swaggerServiceName.$swaggerResourceType"
 
   /**
    * @return A route that returns the swagger resource.
    */
   final def swaggerResourceRoute: Route = {
-    val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath))
+    // Serve Cromwell API docs from either `/swagger/cromwell.yaml` or just `cromwell.yaml`.
+    val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath)) | path(s"$swaggerServiceName.$swaggerResourceType")
 
     def injectBasePath(basePath: Option[String])(response: HttpResponse): HttpResponse = {
       basePath match {
@@ -138,12 +124,10 @@ trait SwaggerResourceHttpService {
       }
     }
 
-    if (swaggerAllOptionsOk) {
-      route ~ options {
-        // Also return status 200 / OK for all OPTIONS requests.
-        complete(StatusCodes.OK)
-      }
-    } else route
+    route ~ options {
+      // Also return status 200 / OK for all OPTIONS requests.
+      complete(StatusCodes.OK)
+    }
   }
 }
 
@@ -151,8 +135,6 @@ trait SwaggerResourceHttpService {
  * Extends the SwaggerUiHttpService and SwaggerResourceHttpService to serve up both.
  */
 trait SwaggerUiResourceHttpService extends SwaggerUiHttpService with SwaggerResourceHttpService {
-  override def swaggerUiDocsPath = swaggerDocsPath
-
   /**
    * @return A route that redirects to the swagger UI and returns the swagger resource.
    */
diff --git a/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala
index f4150094184..f68189ea66e 100644
--- a/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala
+++ b/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala
@@ -69,19 +69,6 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg
       }
   }
 
-  it should "return the index.html" in {
-    Get("/swagger/index.html?url=/swagger/cromwell.yaml") ~>
-      swaggerUiResourceRoute ~>
-      check {
-        assertResult(StatusCodes.OK) {
-          status
-        }
-        assertResult("<!-- HTML for s") {
-          responseAs[String].take(15)
-        }
-      }
-  }
-
   it should "return status OK when getting OPTIONS on paths" in {
     val pathExamples = Table("path", "/", "/swagger", "/swagger/cromwell.yaml", "/swagger/index.html", "/api",
       "/api/workflows/", "/api/workflows/v1", "/workflows/v1/outputs", "/workflows/v1/status",
diff --git a/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala
index 5e53b5677e4..6afcf12bac8 100644
--- a/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala
+++ b/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala
@@ -5,16 +5,13 @@ import akka.http.scaladsl.model.{StatusCodes, Uri}
 import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest}
 import common.assertion.CromwellTimeoutSpec
-import cromwell.webservice.routes.CromwellApiService
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.prop.TableDrivenPropertyChecks
 
 import scala.concurrent.duration._
 
-trait SwaggerUiHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with SwaggerUiHttpService {
-  override def swaggerUiVersion = CromwellApiService.swaggerUiVersion
-}
+trait SwaggerUiHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with SwaggerUiHttpService
 
 trait SwaggerResourceHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with
 TableDrivenPropertyChecks with SwaggerResourceHttpService {
@@ -47,10 +44,18 @@ object SwaggerUiHttpServiceSpec {
 class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec {
   behavior of "SwaggerUiHttpService"
 
-  it should "redirect / to /swagger" in {
-    Get() ~> swaggerUiRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger"))))
+  it should "redirect /swagger to /" in {
+    Get("/swagger") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.MovedPermanently)
+      header("Location") should be(Option(Location(Uri("/"))))
+    }
+  }
+
+  it should "redirect /swagger/index.html?url=/swagger/cromwell.yaml to /" in {
+    Get("/swagger/index.html?url=/swagger/cromwell.yaml") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.MovedPermanently)
+      header("Location") should be(Option(Location(Uri("/"))))
+      responseAs[String] shouldEqual """This and all future requests should be directed to <a href="/">this URI</a>."""
     }
   }
 
@@ -60,16 +65,17 @@ class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec {
     }
   }
 
-  it should "redirect /swagger to the index.html" in {
-    Get("/swagger") ~> swaggerUiRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/api-docs"))))
+  it should "return index.html from the swagger-ui jar for /" in {
+    Get("/") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String].take(15) should be("<!-- HTML for s")
     }
   }
 
-  it should "return index.html from the swagger-ui jar" in {
-    Get("/swagger/index.html") ~> swaggerUiRoute ~> check {
+  it should "return index.html from the swagger-ui jar for base url" in {
+    Get() ~> swaggerUiRoute ~> check {
       status should be(StatusCodes.OK)
+      responseAs[String].take(15) should be("<!-- HTML for s")
     }
   }
 }
@@ -89,36 +95,6 @@ class OverrideBasePathSwaggerUiHttpServiceSpec extends SwaggerResourceHttpServic
   }
 }
 
-class NoRedirectRootSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec {
-  override def swaggerUiFromRoot = false
-
-  behavior of "SwaggerUiHttpService"
-
-  it should "not redirect / to /swagger" in {
-    Get() ~> Route.seal(swaggerUiRoute) ~> check {
-      status should be(StatusCodes.NotFound)
-    }
-  }
-
-  it should "not return options for /" in {
-    Options() ~> Route.seal(swaggerUiRoute) ~> check {
-      status should be(StatusCodes.MethodNotAllowed)
-    }
-  }
-
-  it should "redirect /swagger to the index.html" in {
-    Get("/swagger") ~> swaggerUiRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/api-docs"))))
-    }
-  }
-
-  it should "return index.html from the swagger-ui jar" in {
-    Get("/swagger/index.html") ~> swaggerUiRoute ~> check {
-      status should be(StatusCodes.OK)
-    }
-  }
-}
 
 class YamlSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec {
   override def swaggerServiceName = "testservice"
@@ -190,53 +166,11 @@ class JsonSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec
   }
 }
 
-class NoOptionsSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec {
-  override def swaggerServiceName = "testservice"
-
-  override def swaggerAllOptionsOk = false
-
-  behavior of "SwaggerResourceHttpService"
-
-  it should "service swagger yaml" in {
-    Get("/swagger/testservice.yaml") ~> swaggerResourceRoute ~> check {
-      status should be(StatusCodes.OK)
-      responseAs[String] should startWith("swagger: '2.0'\n")
-    }
-  }
-
-  it should "not service swagger json" in {
-    Get("/swagger/testservice.json") ~> Route.seal(swaggerResourceRoute) ~> check {
-      status should be(StatusCodes.NotFound)
-    }
-  }
-
-  it should "not service /swagger" in {
-    Get("/swagger") ~> Route.seal(swaggerResourceRoute) ~> check {
-      status should be(StatusCodes.NotFound)
-    }
-  }
-
-  it should "not return options for all routes" in {
-    forAll(testPathsForOptions) { path =>
-      Options(path) ~> Route.seal(swaggerResourceRoute) ~> check {
-        status should be(StatusCodes.MethodNotAllowed)
-      }
-    }
-  }
-}
-
 class YamlSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceSpec {
   override def swaggerServiceName = "testservice"
 
   behavior of "SwaggerUiResourceHttpService"
 
-  it should "redirect /swagger to /swagger/index.html with yaml" in {
-    Get("/swagger") ~> swaggerUiResourceRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/swagger/testservice.yaml"))))
-    }
-  }
-
   it should "service swagger yaml" in {
     Get("/swagger/testservice.yaml") ~> swaggerUiResourceRoute ~> check {
       status should be(StatusCodes.OK)
@@ -268,13 +202,6 @@ class JsonSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceS
 
   behavior of "SwaggerUiResourceHttpService"
 
-  it should "redirect /swagger to /swagger/index.html with yaml with json" in {
-    Get("/swagger") ~> swaggerUiResourceRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/swagger/testservice.json"))))
-    }
-  }
-
   it should "service swagger json" in {
     Get("/swagger/testservice.json") ~> swaggerUiResourceRoute ~> check {
       status should be(StatusCodes.OK)

From 938e8367edbe710828a814ac7774b2e513b3aafe Mon Sep 17 00:00:00 2001
From: Janet Gainer-Dewar <jdewar@broadinstitute.org>
Date: Fri, 25 Feb 2022 09:07:15 -0500
Subject: [PATCH 03/19] Delete HCA centaur test (#6690)

---
 .../drs_tests/drs_usa_hca.inputs              |  6 --
 .../drs_tests/drs_usa_hca.wdl                 | 67 -------------------
 .../standardTestCases/drs_usa_hca.test        | 24 -------
 3 files changed, 97 deletions(-)
 delete mode 100644 centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_hca.inputs
 delete mode 100644 centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_hca.wdl
 delete mode 100644 centaur/src/main/resources/standardTestCases/drs_usa_hca.test

diff --git a/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_hca.inputs b/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_hca.inputs
deleted file mode 100644
index b03de55bf32..00000000000
--- a/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_hca.inputs
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-    # Are you here because HCA DRS data moved, again? Do not rage `rm -f` this test without first
-    # reading BT-182 for the multiple reasons why HCA is different from a "hello world"!
-    "drs_usa_hca.file":
-        "drs://jade.datarepo-dev.broadinstitute.org/v1_4641bafb-5190-425b-aea9-9c7b125515c8_e37266ba-790d-4641-aa76-854d94be2fbe"
-}
diff --git a/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_hca.wdl b/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_hca.wdl
deleted file mode 100644
index 35188a69d2e..00000000000
--- a/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_hca.wdl
+++ /dev/null
@@ -1,67 +0,0 @@
-version 1.0
-
-workflow drs_usa_hca {
-    input {
-        File file
-    }
-
-    call localize_drs_with_usa {
-        input:
-            file = file
-    }
-
-    call skip_localize_drs_with_usa { input: file = file }
-
-    output {
-        String path = localize_drs_with_usa.path
-        String hash = localize_drs_with_usa.hash
-        Float size = localize_drs_with_usa.size
-        String cloud = skip_localize_drs_with_usa.path
-    }
-}
-
-task localize_drs_with_usa {
-    input {
-        File file
-    }
-
-    command <<<
-        echo ~{file} > path
-        md5sum ~{file} | cut -c1-32 > hash
-    >>>
-
-    output {
-        String path = read_string("path")
-        String hash = read_string("hash")
-        Float size = size(file)
-    }
-
-    runtime {
-        docker: "ubuntu"
-        backend: "papi-v2-usa"
-        disks: "local-disk 100 HDD"
-    }
-}
-
-task skip_localize_drs_with_usa {
-    input {
-        File file
-    }
-
-    parameter_meta {
-        file: { localization_optional: true }
-    }
-
-    command <<<
-        echo ~{file} > path
-    >>>
-
-    output {
-        String path = read_string("path")
-    }
-
-    runtime {
-        docker: "ubuntu"
-        backend: "papi-v2-usa"
-    }
-}
diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_hca.test b/centaur/src/main/resources/standardTestCases/drs_usa_hca.test
deleted file mode 100644
index f8efd921d21..00000000000
--- a/centaur/src/main/resources/standardTestCases/drs_usa_hca.test
+++ /dev/null
@@ -1,24 +0,0 @@
-name: drs_usa_hca
-testFormat: WorkflowSuccess
-backends: ["papi-v2-usa"]
-skipDescribeEndpointValidation: true
-
-files {
-  workflow: drs_tests/drs_usa_hca.wdl
-  options-dir: "Error: BA-6546 The environment variable CROMWELL_BUILD_RESOURCES_DIRECTORY must be set/export pointing to a valid path such as '${YOUR_CROMWELL_DIR}/target/ci/resources'"
-  options-dir: ${?CROMWELL_BUILD_RESOURCES_DIRECTORY}
-  options: ${files.options-dir}/papi_v2_usa.options.json
-  inputs: drs_tests/drs_usa_hca.inputs
-}
-
-metadata {
-  workflowName: drs_usa_hca
-  status: Succeeded
-
-  "outputs.drs_usa_hca.path" =
-    "/cromwell_root/drs_localization_paths/hca_dev_20201217_test4/5acd55ef/E18_20161004_Neurons_Sample_49_S048_L004_R2_005.fastq.gz"
-  "outputs.drs_usa_hca.hash" = "badf266412ff0e307232421e56d647ed"
-  "outputs.drs_usa_hca.size" = 438932948
-  "outputs.drs_usa_hca.cloud" =
-    "gs://broad-jade-dev-data-bucket/ecb5601e-9026-428c-b49d-3c5f1807ecb7/e37266ba-790d-4641-aa76-854d94be2fbe/E18_20161004_Neurons_Sample_49_S048_L004_R2_005.fastq.gz"
-}

From 3be863823d9723d9f5a013942b8a0f41a7e5b0f3 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@users.noreply.github.com>
Date: Fri, 25 Feb 2022 10:43:46 -0500
Subject: [PATCH 04/19] CromIAM Swagger UI Restructuring  [CROM-6870] (#6684)

---
 .../webservice/CromIamApiService.scala        |   2 -
 .../webservice/SwaggerUiHttpService.scala     | 227 ++++++------------
 .../webservice/SwaggerServiceSpec.scala       |  14 --
 .../webservice/SwaggerUiHttpServiceSpec.scala | 122 ++--------
 4 files changed, 95 insertions(+), 270 deletions(-)

diff --git a/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala b/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala
index 745a3337cec..63694599476 100644
--- a/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala
+++ b/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala
@@ -27,8 +27,6 @@ import scala.concurrent.ExecutionContextExecutor
 
 trait SwaggerService extends SwaggerUiResourceHttpService {
   override def swaggerServiceName = "cromiam"
-  // TODO: Re-common-ize swagger out of cromwell's engine and reuse.
-  override def swaggerUiVersion = "3.23.11" // scala-steward:off
 }
 
 // NB: collection name *must* follow label value rules in cromwell. This needs to be documented somewhere. (although those restrictions are soon to die)
diff --git a/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala b/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala
index 88c3ef1bee5..9fed12ca163 100644
--- a/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala
+++ b/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala
@@ -1,15 +1,14 @@
 package cromiam.webservice
 
 import akka.http.scaladsl.model._
+import akka.http.scaladsl.server
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server._
-import akka.stream.Materializer
-import akka.stream.scaladsl.Sink
+import akka.stream.scaladsl.Flow
 import akka.util.ByteString
+import common.util.VersionUtil
 import cromiam.server.config.SwaggerOauthConfig
 
-import scala.concurrent.{ExecutionContext, Future}
-
 /**
  * Serves up the swagger UI from org.webjars/swagger-ui.
   *
@@ -20,42 +19,21 @@ import scala.concurrent.{ExecutionContext, Future}
   * https://vimeo.com/215325495
  */
 trait SwaggerUiHttpService extends Directives {
-  /**
-   * @return The version of the org.webjars/swagger-ui artifact. For example "2.1.1".
-   */
-  def swaggerUiVersion: String
-
-  /**
-   * Informs the swagger UI of the base of the application url, as hosted on the server.
-   * If your entire app is served under "http://myserver/myapp", then the base URL is "/myapp".
-   * If the app is served at the root of the application, leave this value as the empty string.
-   *
-   * @return The base URL used by the application, or the empty string if there is no base URL. For example "/myapp".
-   */
-  def swaggerUiBaseUrl = ""
-
-  /**
-   * @return The path to the swagger UI html documents. For example "swagger"
-   */
-  def swaggerUiPath = "swagger"
 
-  /**
-   * The path to the actual swagger documentation in either yaml or json, to be rendered by the swagger UI html.
-   *
-   * @return The path to the api documentation to render in the swagger UI.
-   *         For example "api-docs" or "swagger/lenthall.yaml".
-   */
-  def swaggerUiDocsPath = "api-docs"
+  def oauthConfig: SwaggerOauthConfig
 
-  /**
-   * @return When true, if someone requests / (or /baseUrl if setup), redirect to the swagger UI.
-   */
-  def swaggerUiFromRoot = true
+  private lazy val resourceDirectory = {
+    val swaggerUiVersion = VersionUtil.getVersion("swagger-ui", VersionUtil.sbtDependencyVersion("swaggerUi"))
+    s"META-INF/resources/webjars/swagger-ui/$swaggerUiVersion"
+  }
 
-  private def routeFromRoot = get {
-    pathEndOrSingleSlash {
-      // Redirect / to the swagger UI
-      redirect(s"$swaggerUiBaseUrl/$swaggerUiPath", StatusCodes.TemporaryRedirect)
+  private val serveIndex: server.Route = {
+    mapResponseEntity { entityFromJar =>
+      entityFromJar.transformDataBytes(Flow.fromFunction[ByteString, ByteString] { original: ByteString =>
+        ByteString(rewriteSwaggerIndex(original.utf8String))
+      })
+    } {
+      getFromResource(s"$resourceDirectory/index.html")
     }
   }
 
@@ -64,65 +42,65 @@ trait SwaggerUiHttpService extends Directives {
    *
    * @return Route serving the swagger UI.
    */
-  final def swaggerUiRoute = {
-    // when the user hits the doc url, redirect to the index.html with api docs specified on the url
-    val indexRedirect: Route = pathEndOrSingleSlash {
-      redirect(
-        s"$swaggerUiBaseUrl/$swaggerUiPath/index.html?url=$swaggerUiBaseUrl/$swaggerUiDocsPath",
-        StatusCodes.TemporaryRedirect)
-    }
-
-    /* Serve a resource from the swagger-ui webjar/bundle */
-    val resourceServe: Route = getFromResourceDirectory(s"META-INF/resources/webjars/swagger-ui/$swaggerUiVersion")
-
-    /* Mashup of mapResponseWith and mapResponseEntity */
-    def mapResponseEntityWith(f: ResponseEntity => Future[ResponseEntity]): Directive0 = {
-      extractExecutionContext flatMap { implicit executionContext =>
-        mapRouteResultWithPF {
-          case RouteResult.Complete(response) =>
-            f(response.entity) map { entity =>
-              RouteResult.Complete(response.withEntity(entity))
-            }
-        }
+  final def swaggerUiRoute: Route = {
+    pathEndOrSingleSlash {
+      get {
+        serveIndex
       }
-    }
-
-    /* Serve up the index.html, after passing it through a function that rewrites the response. */
-    val indexServe: Route = {
-      pathPrefixTest("index.html") {
-        extractExecutionContext { implicit executionContext =>
-          extractMaterializer { implicit materializer =>
-            mapResponseEntityWith(rewriteSwaggerIndex) {
-              resourceServe
-            }
-          }
+    } ~
+      // We have to be explicit about the paths here since we're matching at the root URL and we don't
+      // want to catch all paths lest we circumvent Spray's not-found and method-not-allowed error
+      // messages.
+      (pathPrefixTest("swagger-ui") | pathPrefixTest("oauth2") | pathSuffixTest("js")
+        | pathSuffixTest("css") | pathPrefixTest("favicon")) {
+        get {
+          getFromResourceDirectory(resourceDirectory)
         }
-      }
-    }
+      } ~
+      // Redirect legacy `/swagger` or `/swagger/index.html?url=/swagger/cromwell.yaml#fragment` requests to the root
+      // URL. The latter form is (somewhat magically) well-behaved in throwing away the `url` query parameter that was
+      // the subject of the CVE linked below while preserving any fragment identifiers to scroll to the right spot in
+      // the Swagger UI.
+      // https://github.com/swagger-api/swagger-ui/security/advisories/GHSA-qrmm-w75w-3wpx
+      (path("swagger" / "index.html") | path("swagger")) {
+        get {
+          redirect("/", StatusCodes.MovedPermanently)
 
-    /* Redirect to the index, serve the rewritten index, or a resource from the swaggerUI webjar. */
-    val route = get {
-      pathPrefix(separateOnSlashes(swaggerUiPath)) {
-        concat(indexRedirect, indexServe, resourceServe)
+        }
       }
-    }
-    if (swaggerUiFromRoot) route ~ routeFromRoot else route
-  }
-
-  private[this] final def rewriteSwaggerIndex(responseEntity: ResponseEntity)
-                                             (implicit
-                                              executionContext: ExecutionContext,
-                                              materializer: Materializer
-                                             ): Future[ResponseEntity] = {
-    val contentType = responseEntity.contentType
-    for {
-      data <- responseEntity.dataBytes.runWith(Sink.head) // Similar to responseEntity.toStrict, but without a timeout
-      replaced = rewriteSwaggerIndex(data.utf8String)
-    } yield HttpEntity.Strict(contentType, ByteString(replaced))
   }
 
   /** Rewrite the swagger index.html. Default passes through the origin data. */
-  protected def rewriteSwaggerIndex(data: String): String = data
+  protected def rewriteSwaggerIndex(original: String): String = {
+    val swaggerOptions =
+      s"""
+        |        validatorUrl: null,
+        |        apisSorter: "alpha",
+        |        oauth2RedirectUrl: window.location.origin + "/swagger/oauth2-redirect.html",
+        |        operationsSorter: "alpha"
+      """.stripMargin
+
+    val initOAuthOriginal = "window.ui = ui"
+
+    val initOAuthReplacement =
+      s"""|
+          |ui.initOAuth({
+          |    clientId: "${oauthConfig.clientId}",
+          |    realm: "${oauthConfig.realm}",
+          |    appName: "${oauthConfig.appName}",
+          |    scopeSeparator: " "
+          |  })
+          |
+          |$initOAuthOriginal
+          |""".stripMargin
+
+
+    original
+      .replace(initOAuthOriginal, initOAuthReplacement)
+      .replace("""url: "https://petstore.swagger.io/v2/swagger.json"""", "url: 'cromiam.yaml'")
+      .replace("""layout: "StandaloneLayout"""", s"""layout: "StandaloneLayout", $swaggerOptions""")
+
+  }
 }
 
 /**
@@ -146,29 +124,17 @@ trait SwaggerResourceHttpService {
    */
   def swaggerResourceType = "yaml"
 
-  /**
-   * Swagger UI sends HTTP OPTIONS before ALL requests, and expects a status 200 / OK. When true (the default) the
-   * swaggerResourceRoute will return 200 / OK for requests for OPTIONS.
-   *
-   * See also:
-   * - https://github.com/swagger-api/swagger-ui/issues/1209
-   * - https://github.com/swagger-api/swagger-ui/issues/161
-   * - https://groups.google.com/forum/#!topic/swagger-swaggersocket/S6_I6FBjdZ8
-   *
-   * @return True if status code 200 should be returned for HTTP OPTIONS requests for the swagger resource.
-   */
-  def swaggerAllOptionsOk = true
-
   /**
    * @return The path to the swagger docs.
    */
-  protected def swaggerDocsPath = s"$swaggerDirectory/$swaggerServiceName.$swaggerResourceType"
+  private lazy val swaggerDocsPath = s"$swaggerDirectory/$swaggerServiceName.$swaggerResourceType"
 
   /**
    * @return A route that returns the swagger resource.
    */
-  final def swaggerResourceRoute = {
-    val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath))
+  final def swaggerResourceRoute: Route = {
+    // Serve CromIAM API docs from either `/swagger/cromiam.yaml` or just `cromiam.yaml`.
+    val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath)) | path(s"$swaggerServiceName.$swaggerResourceType")
     val route = get {
       swaggerDocsDirective {
         // Return /uiPath/serviceName.resourceType from the classpath resources.
@@ -176,12 +142,10 @@ trait SwaggerResourceHttpService {
       }
     }
 
-    if (swaggerAllOptionsOk) {
-      route ~ options {
-        // Also return status 200 / OK for all OPTIONS requests.
-        complete(StatusCodes.OK)
-      }
-    } else route
+    route ~ options {
+      // Also return status 200 / OK for all OPTIONS requests.
+      complete(StatusCodes.OK)
+    }
   }
 }
 
@@ -189,48 +153,9 @@ trait SwaggerResourceHttpService {
  * Extends the SwaggerUiHttpService and SwaggerResourceHttpService to serve up both.
  */
 trait SwaggerUiResourceHttpService extends SwaggerUiHttpService with SwaggerResourceHttpService {
-  override def swaggerUiDocsPath = swaggerDocsPath
-
-  def oauthConfig: SwaggerOauthConfig
 
   /**
    * @return A route that redirects to the swagger UI and returns the swagger resource.
    */
-  final def swaggerUiResourceRoute = swaggerUiRoute ~ swaggerResourceRoute
-
-  override protected def rewriteSwaggerIndex(data: String): String = {
-    // via https://github.com/swagger-api/swagger-ui/tree/v3.2.2#swaggeruibundle
-
-    val bundleOriginal       = """url: "http://petstore.swagger.io/v2/swagger.json","""
-    val bundleOriginalSecure = """url: "https://petstore.swagger.io/v2/swagger.json","""
-
-    val bundleReplacement =
-      s"""|url: "/$swaggerDocsPath",
-          |validatorUrl: null,
-          |oauth2RedirectUrl: window.location.origin + "/$swaggerDirectory/oauth2-redirect.html",
-          |tagsSorter: "alpha",
-          |operationsSorter: "alpha",
-          |""".stripMargin
-
-    // NOTE: Until engine and cromiam can re-common-ize the swagger code, this needs to be sync'ed with
-    //   cromiam.webservice.BasicSwaggerUiHttpServiceSpec.rewriteSwaggerIndex
-    val initOAuthOriginal = "window.ui = ui"
-
-    val initOAuthReplacement =
-      s"""|
-          |ui.initOAuth({
-          |    clientId: "${oauthConfig.clientId}",
-          |    realm: "${oauthConfig.realm}",
-          |    appName: "${oauthConfig.appName}",
-          |    scopeSeparator: " "
-          |  })
-          |
-          |$initOAuthOriginal
-          |""".stripMargin
-
-    data
-      .replace(initOAuthOriginal, initOAuthReplacement)
-      .replace(bundleOriginal, bundleReplacement)
-      .replace(bundleOriginalSecure, bundleReplacement)
-  }
+  final def swaggerUiResourceRoute: Route = swaggerUiRoute ~ swaggerResourceRoute
 }
diff --git a/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala
index 9ba70266162..9bc6fddc179 100644
--- a/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala
+++ b/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala
@@ -77,20 +77,6 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg
       }
   }
 
-  it should "return the index.html" in {
-    Get("/swagger/index.html") ~>
-      swaggerUiResourceRoute ~>
-      check {
-        assertResult(StatusCodes.OK) {
-          status
-        }
-        assertResult("<!-- HTML for static distribution bundle build -->") {
-          responseAs[String].take(50)
-        }
-        assertResult(ContentTypes.`text/html(UTF-8)`)(contentType)
-      }
-  }
-
   it should "return status OK when getting OPTIONS on paths" in {
     val pathExamples = Table("path", "/", "/swagger", "/swagger/cromwell.yaml", "/swagger/index.html", "/api",
       "/api/workflows/", "/api/workflows/v1", "/workflows/v1/outputs", "/workflows/v1/status",
diff --git a/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala
index e95b2fda630..1d0103a09fd 100644
--- a/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala
+++ b/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala
@@ -6,7 +6,6 @@ import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.testkit.ScalatestRouteTest
 import common.assertion.CromwellTimeoutSpec
 import cromiam.server.config.SwaggerOauthConfig
-import cromiam.webservice.SwaggerUiHttpServiceSpec._
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.prop.TableDrivenPropertyChecks
@@ -14,8 +13,6 @@ import org.scalatest.prop.TableDrivenPropertyChecks
 
 trait SwaggerUiHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with SwaggerUiHttpService {
   def actorRefFactory = system
-
-  override def swaggerUiVersion = TestSwaggerUiVersion
 }
 
 trait SwaggerResourceHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with
@@ -46,47 +43,18 @@ class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec {
   // Replace same magic string used in SwaggerUiResourceHttpService.rewriteSwaggerIndex
     data.replace("window.ui = ui", "replaced-client-id")
 
-  it should "redirect / to /swagger" in {
-    Get() ~> swaggerUiRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger"))))
-      contentType should be(ContentTypes.`text/html(UTF-8)`)
-    }
-  }
-
-  it should "not return options for /" in {
-    Options() ~> Route.seal(swaggerUiRoute) ~> check {
-      status should be(StatusCodes.MethodNotAllowed)
-      contentType should be(ContentTypes.`text/plain(UTF-8)`)
-    }
-  }
-
-  it should "redirect /swagger to the index.html" in {
+  it should "redirect /swagger to /" in {
     Get("/swagger") ~> swaggerUiRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/api-docs"))))
-      contentType should be(ContentTypes.`text/html(UTF-8)`)
+      status should be(StatusCodes.MovedPermanently)
+      header("Location") should be(Option(Location(Uri("/"))))
     }
   }
 
-  it should "return index.html from the swagger-ui jar" in {
-    Get("/swagger/index.html") ~> swaggerUiRoute ~> check {
-      status should be(StatusCodes.OK)
-      responseAs[String] should include("replaced-client-id")
-      contentType should be(ContentTypes.`text/html(UTF-8)`)
-    }
-  }
-}
-
-class NoRedirectRootSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec {
-  override def swaggerUiFromRoot = false
-
-  behavior of "SwaggerUiHttpService"
-
-  it should "not redirect / to /swagger" in {
-    Get() ~> Route.seal(swaggerUiRoute) ~> check {
-      status should be(StatusCodes.NotFound)
-      contentType should be(ContentTypes.`text/plain(UTF-8)`)
+  it should "redirect /swagger/index.html?url=/swagger/cromiam.yaml to /" in {
+    Get("/swagger/index.html?url=/swagger/cromiam.yaml") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.MovedPermanently)
+      header("Location") should be(Option(Location(Uri("/"))))
+      responseAs[String] shouldEqual """This and all future requests should be directed to <a href="/">this URI</a>."""
     }
   }
 
@@ -97,20 +65,23 @@ class NoRedirectRootSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec {
     }
   }
 
-  it should "redirect /swagger to the index.html" in {
-    Get("/swagger") ~> swaggerUiRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/api-docs"))))
-      contentType should be(ContentTypes.`text/html(UTF-8)`)
+  it should "return index.html from the swagger-ui jar for /" in {
+    Get("/") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String].take(15) should be("<!-- HTML for s")
     }
   }
 
-  it should "return index.html from the swagger-ui jar" in {
-    Get("/swagger/index.html") ~> swaggerUiRoute ~> check {
+  it should "return index.html from the swagger-ui jar for base url" in {
+    Get() ~> swaggerUiRoute ~> check {
       status should be(StatusCodes.OK)
-      contentType should be(ContentTypes.`text/html(UTF-8)`)
+      responseAs[String].take(15) should be("<!-- HTML for s")
     }
   }
+
+  override def oauthConfig: SwaggerOauthConfig = SwaggerOauthConfig(
+    clientId = "test-client-id", realm = "test-realm", appName = "test-appname"
+  )
 }
 
 class YamlSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec {
@@ -191,45 +162,6 @@ class JsonSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec
   }
 }
 
-class NoOptionsSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec {
-  override def swaggerServiceName = "testservice"
-
-  override def swaggerAllOptionsOk = false
-
-  behavior of "SwaggerResourceHttpService"
-
-  it should "service swagger yaml" in {
-    Get("/swagger/testservice.yaml") ~> swaggerResourceRoute ~> check {
-      status should be(StatusCodes.OK)
-      responseAs[String] should startWith("swagger: '2.0'\n")
-      contentType should be(ContentTypes.`application/octet-stream`)
-    }
-  }
-
-  it should "not service swagger json" in {
-    Get("/swagger/testservice.json") ~> Route.seal(swaggerResourceRoute) ~> check {
-      status should be(StatusCodes.NotFound)
-      contentType should be(ContentTypes.`text/plain(UTF-8)`)
-    }
-  }
-
-  it should "not service /swagger" in {
-    Get("/swagger") ~> Route.seal(swaggerResourceRoute) ~> check {
-      status should be(StatusCodes.NotFound)
-      contentType should be(ContentTypes.`text/plain(UTF-8)`)
-    }
-  }
-
-  it should "not return options for all routes" in {
-    forAll(testPathsForOptions) { path =>
-      Options(path) ~> Route.seal(swaggerResourceRoute) ~> check {
-        status should be(StatusCodes.MethodNotAllowed)
-        contentType should be(ContentTypes.`text/plain(UTF-8)`)
-      }
-    }
-  }
-}
-
 class YamlSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceSpec {
 
   override def oauthConfig: SwaggerOauthConfig = SwaggerOauthConfig("clientId", "realm", "appName")
@@ -237,14 +169,6 @@ class YamlSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceS
 
   behavior of "SwaggerUiResourceHttpService"
 
-  it should "redirect /swagger to /swagger/index.html with yaml" in {
-    Get("/swagger") ~> swaggerUiResourceRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/swagger/testservice.yaml"))))
-      contentType should be(ContentTypes.`text/html(UTF-8)`)
-    }
-  }
-
   it should "service swagger yaml" in {
     Get("/swagger/testservice.yaml") ~> swaggerUiResourceRoute ~> check {
       status should be(StatusCodes.OK)
@@ -281,14 +205,6 @@ class JsonSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceS
 
   behavior of "SwaggerUiResourceHttpService"
 
-  it should "redirect /swagger to /swagger/index.html with yaml with json" in {
-    Get("/swagger") ~> swaggerUiResourceRoute ~> check {
-      status should be(StatusCodes.TemporaryRedirect)
-      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/swagger/testservice.json"))))
-      contentType should be(ContentTypes.`text/html(UTF-8)`)
-    }
-  }
-
   it should "service swagger json" in {
     Get("/swagger/testservice.json") ~> swaggerUiResourceRoute ~> check {
       status should be(StatusCodes.OK)

From af84cc7a694fe639f0fd52d9855099a61dda2246 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@users.noreply.github.com>
Date: Mon, 28 Feb 2022 10:24:03 -0500
Subject: [PATCH 05/19] Swagger UI 3.23.11 => 4.5.0 [BT-562] (#6691)

---
 project/Dependencies.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 13f7e7746c3..78b6f81bce2 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -126,7 +126,7 @@ object Dependencies {
   private val sprayJsonV = "1.3.6"
   private val sttpV = "1.5.19" // scala-steward:off (CROM-6564)
   private val swaggerParserV = "1.0.56"
-  private val swaggerUiV = "3.23.11" // scala-steward:off (CROM-6621)
+  private val swaggerUiV = "4.5.0"
   private val testContainersScalaV = "0.39.8"
   private val tikaV = "2.1.0"
   private val typesafeConfigV = "1.4.1"

From 05114305d790553c0bfc460b9eabe0e2b7fbdaa4 Mon Sep 17 00:00:00 2001
From: Saloni Shah <salonipshah11@gmail.com>
Date: Tue, 1 Mar 2022 18:57:16 -0500
Subject: [PATCH 06/19] Pick up workflows to run from hog group with lowest
 actively running workflows [BW-568] (#6680)

---
 .../slick/WorkflowStoreSlickDatabase.scala    |  24 ++-
 .../tables/WorkflowStoreEntryComponent.scala  |  95 +++++++++--
 .../workflowstore/SqlWorkflowStoreSpec.scala  | 151 +++++++++++++++---
 .../cromwell/services/database/LobSpec.scala  |   6 +-
 4 files changed, 232 insertions(+), 44 deletions(-)

diff --git a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
index 1ded96f6066..a39dcbf4195 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
@@ -88,13 +88,25 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase {
       } yield ()
     }
 
+    def fetchAndUpdateStartableWfs(hogGroup: Option[String]) = {
+      for {
+        workflowStoreEntries <- dataAccess.fetchStartableWfsForHogGroup(
+          (limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded, hogGroup)
+        ).result
+        _ <- DBIO.sequence(
+          workflowStoreEntries map updateForFetched(cromwellId, heartbeatTimestampTo, workflowStateFrom, workflowStateTo)
+        )
+      } yield workflowStoreEntries
+    }
+
     val action = for {
-      workflowStoreEntries <- dataAccess.fetchStartableWorkflows(
-        limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded, excludedGroups
-      ).result
-      _ <- DBIO.sequence(
-        workflowStoreEntries map updateForFetched(cromwellId, heartbeatTimestampTo, workflowStateFrom, workflowStateTo)
-      )
+      // find hog group with lowest count of actively running workflows
+      hogGroupOption <- dataAccess.getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut, workflowStateExcluded, excludedGroups).result.headOption
+      workflowStoreEntries <- hogGroupOption match {
+        // if no such hog group was found, all hog groups have workflows that are either actively running or in "OnHold" status
+        case None => DBIO.successful(Seq.empty[WorkflowStoreEntry])
+        case Some(hogGroup) => fetchAndUpdateStartableWfs(hogGroup)
+      }
     } yield workflowStoreEntries
 
     // This should be safe because there are no repeated reads and the same rows are being locked for update as with
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index 0b0bf2c510a..7971f177849 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -73,31 +73,92 @@ trait WorkflowStoreEntryComponent {
   )
 
   /**
-    * Returns up to "limit" startable workflows, sorted by submission time.
-    */
-  def fetchStartableWorkflows(limit: Long,
-     heartbeatTimestampTimedOut: Timestamp,
-     excludeWorkflowState: String,
-     excludedGroups: Set[String]
-    ): Query[WorkflowStoreEntries, WorkflowStoreEntry, Seq] = {
-      val query = for {
-        row <- workflowStoreEntries
-        /*
+   * Return hog group with the lowest count of actively running workflows
+   */
+  def getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut: Timestamp,
+                                      excludeWorkflowState: String,
+                                      excludedGroups: Set[String]): Query[Rep[Option[String]], Option[String], Seq] = {
+    val startableWorkflows = for {
+      row <- workflowStoreEntries
+      /*
         This looks for:
-
         1) Workflows with no heartbeat (newly submitted or from a cleanly shut down Cromwell).
         2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell.
+        3) Workflows not in "OnHold" state
+        4) Workflows that don't belong to hog groups in excludedGroups
+      */
+      if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) &&
+        (row.workflowState =!= excludeWorkflowState) &&
+        !(row.hogGroup inSet excludedGroups)
+    } yield row
+
+    // calculates the count of startable workflows per hog group
+    val numOfStartableWfsByHogGroup = startableWorkflows
+      .groupBy(_.hogGroup)
+      .map { case (hogGroupName, groups) => (hogGroupName, groups.length) }
+      .sortBy(_._2.asc)
+
+    val totalWorkflows = for {
+      row <- workflowStoreEntries
+      /*
+        This looks for:
+        1) Workflows not in "OnHold" state
+        2) Workflows that don't belong to hog groups in excludedGroups
+      */
+      if row.workflowState =!= excludeWorkflowState &&
+        !(row.hogGroup inSet excludedGroups)
+    } yield row
+
+    // calculates the count of total workflows per hog group
+    val totalWorkflowsByHogGroup = totalWorkflows
+      .groupBy(_.hogGroup)
+      .map { case (hogGroupName, groups) => (hogGroupName, groups.length) }
+      .sortBy(_._2.asc)
+
+    // calculates the number of actively running workflows for each hog group. If a hog group
+    // has all it's workflows that are either actively running or in "OnHold" status it is not
+    // included in the list. Hog groups that have no workflows actively running return count as 0
+    val wfsRunningPerHogGroup = for {
+      (t_group, t_ct) <- totalWorkflowsByHogGroup
+      (s_group, s_ct) <- numOfStartableWfsByHogGroup if t_group === s_group
+    } yield (t_group, t_ct - s_ct)
+
+    // sort the above calculated result set first by the count of actively running workflows and then sort it
+    // alphabetically by hog group. Then take the first row of the result and return the hog group name.
+    wfsRunningPerHogGroup.sortBy { case (hogGroupName, ct) => (ct.asc, hogGroupName) }.take(1).map(_._1)
+  }
 
-        Workflows are taken by submission time, oldest first. This is a "query for update", meaning rows are
-        locked such that readers are blocked since we will do an update subsequent to this select in the same
-        transaction that we know will impact those readers.
-         */
+  /**
+   * Returns up to "limit" startable workflows, sorted by submission time, that belong to
+   * given hog group and are not in "OnHold" status.
+   */
+  val fetchStartableWfsForHogGroup = Compiled(
+    (limit: ConstColumn[Long],
+     heartbeatTimestampTimedOut: ConstColumn[Timestamp],
+     excludeWorkflowState: Rep[String],
+     hogGroup: Rep[Option[String]]) => {
+
+      val workflowsToStart = for {
+        row <- workflowStoreEntries
+        /*
+          This looks for:
+          1) Workflows with no heartbeat (newly submitted or from a cleanly shut down Cromwell).
+          2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell.
+          3) Workflows not in "OnHold" state
+          4) Workflows that belong to included hog group
+        */
         if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) &&
           (row.workflowState =!= excludeWorkflowState) &&
-          !(row.hogGroup inSet excludedGroups)
+          (row.hogGroup === hogGroup)
       } yield row
-      query.forUpdate.sortBy(_.submissionTime.asc).take(limit)
+
+      /*
+        This is a "query for update", meaning rows are locked such that readers are blocked since we will
+        do an update subsequent to this select in the same transaction that we know will impact those readers.
+       */
+      workflowsToStart.forUpdate.sortBy(_.submissionTime.asc).take(limit)
     }
+  )
 
   /**
     * Useful for counting workflows in a given state.
diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
index af49ef5149f..223d9ac0034 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
@@ -13,17 +13,16 @@ import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
-import org.scalatest.enablers.Emptiness._
 import org.scalatest.time.{Millis, Seconds, Span}
 import org.specs2.mock.Mockito
 import spray.json.{JsObject, JsString}
 
-import scala.concurrent.{ExecutionContext, Future}
 import scala.concurrent.duration._
+import scala.concurrent.{Await, ExecutionContext, Future}
 
 class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalaFutures with BeforeAndAfterAll with Mockito {
   implicit val ec = ExecutionContext.global
-  implicit val defaultPatience = PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(100, Millis)))
+  implicit val defaultPatience = PatienceConfig(scaled(Span(20, Seconds)), scaled(Span(100, Millis)))
 
   val onHoldSourceFilesCollection = NonEmptyList.of(
     WorkflowSourceFilesCollection(
@@ -121,6 +120,16 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     lazy val workflowStore = SqlWorkflowStore(dataAccess, metadataDataAccess)
 
+    def updateWfToRunning(startableWorkflows: List[WorkflowToStart]): Unit = {
+      startableWorkflows.foreach { wf =>
+        Await.result(workflowStore.sqlDatabase.updateWorkflowState(
+          wf.id.toString,
+          WorkflowStoreState.Submitted.toString,
+          WorkflowStoreState.Running.toString
+        ), 5.seconds)
+      }
+    }
+
     it should "start container if required" taggedAs DbmsTest in {
       containerOpt.foreach {
         _.start
@@ -244,22 +253,128 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       } yield ()).futureValue
     }
 
-    it should "select appropriately with the excludedGroups parameter" taggedAs DbmsTest in {
+    it should "start workflows from hog group with lowest count of running workflows" taggedAs DbmsTest in {
+      // first submission of 50 workflows for hogGroup "Goldfinger"
+      val goldFingerWorkflowIds = (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection1), 5.seconds)).flatMap(_.map(_.id).toList)
+
+      // second submission of 50 workflows for hogGroup "Highlander"
+      val highlanderWorkflowIds = (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection2), 5.seconds)).flatMap(_.map(_.id).toList)
+
+      for (_ <- 1 to 10) yield {
+        (for {
+          // since both hog groups have 0 workflows running, the hog groups are sorted alphabetically and first one is picked
+          startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
+          _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+          _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+          _ = updateWfToRunning(startableWorkflows1)
+
+          startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
+          _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander")
+          _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
+          _ = updateWfToRunning(startableWorkflows2)
+        } yield ()).futureValue
+      }
+
+      // remove entries from WorkflowStore
+      (goldFingerWorkflowIds ++ highlanderWorkflowIds).foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
+    }
+
+    it should "respect excludedHogGroups and start workflows from hog group with lowest count of running workflows" taggedAs DbmsTest in {
+      (for {
+        // first submission of 10 workflows for hogGroup "Goldfinger"
+        goldFingerSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1))
+        goldFingerWorkflowIds = goldFingerSubmissions.flatMap(_.map(_.id).toList)
+
+        // second submission of 10 workflows for hogGroup "Zardoz"
+        zardozSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(excludedGroupSourceFilesCollection))
+        zardozWorkflowIds = zardozSubmissions.flatMap(_.map(_.id).toList)
+
+        startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
+        _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+        _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows1)
+
+        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
+        _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+        _ = startableWorkflows2.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows2)
+
+        // there are 10 workflows from hog group "Zardoz" in the store, but since the group is excluded, 0 workflows are returned here
+        startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
+        _ = startableWorkflows3.size should be(0)
+
+        // hog group "Zardoz" has tokens to run workflows, hence don't exclude it
+        startableWorkflows4 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
+        _ = startableWorkflows4.map(_.hogGroup.value).toSet.head should be("Zardoz")
+        _ = startableWorkflows4.map(_.id).foreach(x => zardozWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows4)
+
+        startableWorkflows5 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
+        _ = startableWorkflows5.map(_.hogGroup.value).toSet.head should be("Zardoz")
+        _ = startableWorkflows5.map(_.id).foreach(x => zardozWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows5)
+
+        // remove entries from WorkflowStore
+        workflowsList = goldFingerWorkflowIds ++ zardozWorkflowIds
+        _ = workflowsList.foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
+      } yield()).futureValue
+    }
+
+    it should "start workflows from hog group with lowest count of running workflows for multiple hog groups" taggedAs DbmsTest in {
       (for {
-        submissionResponsesExcluded <- workflowStore.add(excludedGroupSourceFilesCollection)
-        submissionResponsesIncluded1 <- workflowStore.add(includedGroupSourceFilesCollection1)
-        submissionResponsesIncluded2 <- workflowStore.add(includedGroupSourceFilesCollection2)
-        submissionResponsesIncluded3 <- workflowStore.add(includedGroupSourceFilesCollection3)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(3, "A08", 1.second, excludedGroups = Set("Zardoz"))
-        _ = startableWorkflows.map(_.id).intersect(submissionResponsesExcluded.map(_.id).toList).size should be(0)
-        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded1.map(_.id).toList).size should be(1)
-        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded2.map(_.id).toList).size should be(1)
-        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded3.map(_.id).toList).size should be(1)
-        _ = startableWorkflows.map(_.id).size should be(3)
-        _ <- workflowStore.deleteFromStore(submissionResponsesExcluded.head.id) // Tidy up
-        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded1.head.id) // Tidy up
-        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded2.head.id) // Tidy up
-        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded3.head.id) // Tidy up
+        // first submission of 10 workflows for hogGroup "Goldfinger"
+        goldFingerSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1))
+        goldFingerWorkflowIds = goldFingerSubmissions.flatMap(_.map(_.id).toList)
+
+        // second submission of 10 workflows for hogGroup "Highlander"
+        highlanderSubmissions <- Future.sequence(for (_ <- 1 to 15) yield workflowStore.add(includedGroupSourceFilesCollection2))
+        highlanderWorkflowIds = highlanderSubmissions.flatMap(_.map(_.id).toList)
+
+        // since both hog groups have 0 workflows running, the hog groups are sorted alphabetically and first one is picked
+        startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+        _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows1)
+
+        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander")
+        _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows2)
+
+        // new submission for hog group "Finding Forrester"
+        foresterSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection3))
+        foresterWorkflowIds = foresterSubmissions.flatMap(_.map(_.id).toList)
+
+        // now hog group "Finding Forrester" has 0 workflows running, hence it is picked to run
+        startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows3.map(_.hogGroup.value).toSet.head should be("Finding Forrester")
+        _ = startableWorkflows3.map(_.id).foreach(x => foresterWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows3)
+
+        // since all 3 hog groups have 5 workflows running each, the hog groups are sorted alphabetically and first one is picked
+        startableWorkflows4 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows4.map(_.hogGroup.value).toSet.head should be("Finding Forrester")
+        _ = startableWorkflows4.map(_.id).foreach(x => foresterWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows4)
+
+        startableWorkflows5 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows5.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+        _ = startableWorkflows5.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows5)
+
+        startableWorkflows6 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows6.map(_.hogGroup.value).toSet.head should be("Highlander")
+        _ = startableWorkflows6.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows6)
+
+        startableWorkflows7 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows7.map(_.hogGroup.value).toSet.head should be("Highlander")
+        _ = startableWorkflows7.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows7)
+
+        // remove entries from WorkflowStore
+        workflowsList = goldFingerWorkflowIds ++ highlanderWorkflowIds ++ foresterWorkflowIds
+        _ = workflowsList.foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
       } yield ()).futureValue
     }
 
diff --git a/services/src/test/scala/cromwell/services/database/LobSpec.scala b/services/src/test/scala/cromwell/services/database/LobSpec.scala
index 74a3cc619d5..042d901eab2 100644
--- a/services/src/test/scala/cromwell/services/database/LobSpec.scala
+++ b/services/src/test/scala/cromwell/services/database/LobSpec.scala
@@ -122,7 +122,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = Option(Array.empty[Byte]).toBlobOption,
         customLabels = clob,
-        hogGroup = None,
+        hogGroup = Option("abc-1"),
       )
 
       val noneWorkflowUuid = WorkflowId.randomId().toString
@@ -141,7 +141,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = None,
         customLabels = clob,
-        hogGroup = None,
+        hogGroup = Option("abc-1"),
       )
 
       val aByte = 'a'.toByte
@@ -161,7 +161,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = Option(Array(aByte)).toBlobOption,
         customLabels = clob,
-        hogGroup = None,
+        hogGroup = Option("abc-1"),
       )
 
       val workflowStoreEntries = Seq(emptyWorkflowStoreEntry, noneWorkflowStoreEntry, aByteWorkflowStoreEntry)

From 5616d5b03b458c008271da0776cee7460b7acff7 Mon Sep 17 00:00:00 2001
From: Katrina P <68349264+kpierre13@users.noreply.github.com>
Date: Mon, 7 Mar 2022 09:59:35 -0500
Subject: [PATCH 07/19] [BW-1090] - Disable CWL tests due to PAPI 503 flakiness
 (#6694)

* [BW-1090] - Disable CWL tests due to PAPI 503 flakiness

* Removing blink due to Travis web UI incompatibility

* Just echoing
---
 src/ci/bin/testConformancePapiV2beta.sh | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/src/ci/bin/testConformancePapiV2beta.sh b/src/ci/bin/testConformancePapiV2beta.sh
index cb952694b4e..7117a2d78ba 100755
--- a/src/ci/bin/testConformancePapiV2beta.sh
+++ b/src/ci/bin/testConformancePapiV2beta.sh
@@ -1,21 +1,4 @@
 #!/usr/bin/env bash
 
-set -o errexit -o nounset -o pipefail
-export CROMWELL_BUILD_REQUIRES_SECURE=true
-# import in shellcheck / CI / IntelliJ compatible ways
-# shellcheck source=/dev/null
-source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
-# shellcheck source=/dev/null
-source "${BASH_SOURCE%/*}/test_papi.inc.sh" || source test_papi.inc.sh
-
-cromwell::build::setup_common_environment
-
-cromwell::build::setup_conformance_environment
-
-cromwell::build::papi::setup_papi_conformance_environment
-
-cromwell::build::assemble_jars
-
-cromwell::build::run_conformance
-
-cromwell::build::generate_code_coverage
+echo "Test is temporarily disabled due to ongoing PAPI 503 flakiness."
+exit 0
\ No newline at end of file

From 713a5dbc597e3b09fb77698e8c86c76e9e6bb419 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@users.noreply.github.com>
Date: Tue, 8 Mar 2022 15:54:39 -0500
Subject: [PATCH 08/19] Revert changes introducing workflow pickup failures
 [BW-1128] (#6697)

* Revert "Pick up workflows to run from hog group with lowest actively running workflows [BW-568] (#6680)"

This reverts commit 05114305d790553c0bfc460b9eabe0e2b7fbdaa4.

* Revert "Option to exclude groups from workflow pickup [BW-1061] (#6668)"

This reverts commit bfef756ca35b46570dff3fda57f77dd4b2b0d25c.
---
 .../slick/WorkflowStoreSlickDatabase.scala    |  27 +-
 .../tables/WorkflowStoreEntryComponent.scala  |  95 +-----
 .../sql/WorkflowStoreSqlDatabase.scala        |   3 +-
 .../workflow/WorkflowManagerActor.scala       |   2 +-
 .../workflowstore/InMemoryWorkflowStore.scala |   6 +-
 .../workflowstore/SqlWorkflowStore.scala      |   5 +-
 .../workflowstore/WorkflowStore.scala         |   2 +-
 .../workflowstore/WorkflowStoreAccess.scala   |  10 +-
 .../workflowstore/WorkflowStoreActor.scala    |   2 +-
 .../WorkflowStoreCoordinatedAccessActor.scala |   6 +-
 .../WorkflowStoreEngineActor.scala            |   8 +-
 .../workflowstore/SqlWorkflowStoreSpec.scala  | 301 +++---------------
 ...kflowStoreCoordinatedAccessActorSpec.scala |  12 +-
 .../engine/WorkflowStoreActorSpec.scala       |  10 +-
 .../cromwell/services/database/LobSpec.scala  |   9 +-
 15 files changed, 95 insertions(+), 403 deletions(-)

diff --git a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
index a39dcbf4195..5d88d0dbec8 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
@@ -59,8 +59,7 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase {
                                      heartbeatTimestampTo: Timestamp,
                                      workflowStateFrom: String,
                                      workflowStateTo: String,
-                                     workflowStateExcluded: String,
-                                     excludedGroups: Set[String])
+                                     workflowStateExcluded: String)
                                     (implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]] = {
 
     def updateForFetched(cromwellId: String,
@@ -88,25 +87,13 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase {
       } yield ()
     }
 
-    def fetchAndUpdateStartableWfs(hogGroup: Option[String]) = {
-      for {
-        workflowStoreEntries <- dataAccess.fetchStartableWfsForHogGroup(
-          (limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded, hogGroup)
-        ).result
-        _ <- DBIO.sequence(
-          workflowStoreEntries map updateForFetched(cromwellId, heartbeatTimestampTo, workflowStateFrom, workflowStateTo)
-        )
-      } yield workflowStoreEntries
-    }
-
     val action = for {
-      // find hog group with lowest count of actively running workflows
-      hogGroupOption <- dataAccess.getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut, workflowStateExcluded, excludedGroups).result.headOption
-      workflowStoreEntries <- hogGroupOption match {
-        // if no such hog group was found, all hog groups have workflows that are either actively running or in "OnHold" status
-        case None => DBIO.successful(Seq.empty[WorkflowStoreEntry])
-        case Some(hogGroup) => fetchAndUpdateStartableWfs(hogGroup)
-      }
+      workflowStoreEntries <- dataAccess.fetchStartableWorkflows(
+        (limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded)
+      ).result
+      _ <- DBIO.sequence(
+        workflowStoreEntries map updateForFetched(cromwellId, heartbeatTimestampTo, workflowStateFrom, workflowStateTo)
+      )
     } yield workflowStoreEntries
 
     // This should be safe because there are no repeated reads and the same rows are being locked for update as with
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index 7971f177849..3fb6ec48cdd 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -73,90 +73,29 @@ trait WorkflowStoreEntryComponent {
   )
 
   /**
-   * Return hog group with the lowest count of actively running workflows
-   */
-  def getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut: Timestamp,
-                                      excludeWorkflowState: String,
-                                      excludedGroups: Set[String]): Query[Rep[Option[String]], Option[String], Seq] = {
-    val startableWorkflows = for {
-      row <- workflowStoreEntries
-      /*
-        This looks for:
-        1) Workflows with no heartbeat (newly submitted or from a cleanly shut down Cromwell).
-        2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell.
-        3) Workflows not in "OnHold" state
-        4) Workflows that don't belong to hog groups in excludedGroups
-      */
-      if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) &&
-        (row.workflowState =!= excludeWorkflowState) &&
-        !(row.hogGroup inSet excludedGroups)
-    } yield row
-
-    // calculates the count of startable workflows per hog group
-    val numOfStartableWfsByHogGroup = startableWorkflows
-      .groupBy(_.hogGroup)
-      .map { case (hogGroupName, groups) => (hogGroupName, groups.length) }
-      .sortBy(_._2.asc)
-
-    val totalWorkflows = for {
-      row <- workflowStoreEntries
-      /*
-        This looks for:
-        1) Workflows not in "OnHold" state
-        2) Workflows that don't belong to hog groups in excludedGroups
-      */
-      if row.workflowState =!= excludeWorkflowState &&
-        !(row.hogGroup inSet excludedGroups)
-    } yield row
-
-    // calculates the count of total workflows per hog group
-    val totalWorkflowsByHogGroup = totalWorkflows
-      .groupBy(_.hogGroup)
-      .map { case (hogGroupName, groups) => (hogGroupName, groups.length) }
-      .sortBy(_._2.asc)
-
-    // calculates the number of actively running workflows for each hog group. If a hog group
-    // has all it's workflows that are either actively running or in "OnHold" status it is not
-    // included in the list. Hog groups that have no workflows actively running return count as 0
-    val wfsRunningPerHogGroup = for {
-      (t_group, t_ct) <- totalWorkflowsByHogGroup
-      (s_group, s_ct) <- numOfStartableWfsByHogGroup if t_group === s_group
-    } yield (t_group, t_ct - s_ct)
-
-    // sort the above calculated result set first by the count of actively running workflows and then sort it
-    // alphabetically by hog group. Then take the first row of the result and return the hog group name.
-    wfsRunningPerHogGroup.sortBy { case (hogGroupName, ct) => (ct.asc, hogGroupName) }.take(1).map(_._1)
-  }
-
-  /**
-   * Returns up to "limit" startable workflows, sorted by submission time, that belong to
-   * given hog group and are not in "OnHold" status.
-   */
-  val fetchStartableWfsForHogGroup = Compiled(
+    * Returns up to "limit" startable workflows, sorted by submission time.
+    */
+  val fetchStartableWorkflows = Compiled(
     (limit: ConstColumn[Long],
      heartbeatTimestampTimedOut: ConstColumn[Timestamp],
-     excludeWorkflowState: Rep[String],
-     hogGroup: Rep[Option[String]]) => {
-
-      val workflowsToStart = for {
+     excludeWorkflowState: Rep[String]
+    ) => {
+      val query = for {
         row <- workflowStoreEntries
         /*
-          This looks for:
-          1) Workflows with no heartbeat (newly submitted or from a cleanly shut down Cromwell).
-          2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell.
-          3) Workflows not in "OnHold" state
-          4) Workflows that belong to included hog group
-        */
+        This looks for:
+
+        1) Workflows with no heartbeat (newly submitted or from a cleanly shut down Cromwell).
+        2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell.
+
+        Workflows are taken by submission time, oldest first. This is a "query for update", meaning rows are
+        locked such that readers are blocked since we will do an update subsequent to this select in the same
+        transaction that we know will impact those readers.
+         */
         if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) &&
-          (row.workflowState =!= excludeWorkflowState) &&
-          (row.hogGroup === hogGroup)
+          (row.workflowState =!= excludeWorkflowState)
       } yield row
-
-      /*
-        This is a "query for update", meaning rows are locked such that readers are blocked since we will
-        do an update subsequent to this select in the same transaction that we know will impact those readers.
-       */
-      workflowsToStart.forUpdate.sortBy(_.submissionTime.asc).take(limit)
+      query.forUpdate.sortBy(_.submissionTime.asc).take(limit)
     }
   )
 
diff --git a/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala
index 82dc51c875b..a8312f6d685 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala
@@ -74,8 +74,7 @@ ____    __    ____  ______   .______       __  ___  _______  __        ______
                             heartbeatTimestampTo: Timestamp,
                             workflowStateFrom: String,
                             workflowStateTo: String,
-                            workflowStateExcluded: String,
-                            excludedGroups: Set[String])
+                            workflowStateExcluded: String)
                            (implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]]
 
   def writeWorkflowHeartbeats(workflowExecutionUuids: Seq[String],
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index d4b552f15ef..b2bd7c53f1c 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -173,7 +173,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
         Determine the number of available workflow slots and request the smaller of that number and maxWorkflowsToLaunch.
        */
       val maxNewWorkflows = maxWorkflowsToLaunch min (maxWorkflowsRunning - stateData.workflows.size - stateData.subWorkflows.size)
-      params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows, excludedGroups = Set.empty)
+      params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows)
       stay()
     case Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart, _) =>
       log.debug("WorkflowStore provided no new workflows to start")
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala
index b6913f6172c..121371b162c 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala
@@ -1,6 +1,7 @@
 package cromwell.engine.workflow.workflowstore
 
 import java.time.OffsetDateTime
+
 import cats.data.NonEmptyList
 import cromwell.core.{HogGroup, WorkflowId, WorkflowSourceFilesCollection}
 import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreAbortResponse.WorkflowStoreAbortResponse
@@ -31,10 +32,7 @@ class InMemoryWorkflowStore extends WorkflowStore {
     * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp
     * flag to true
     */
-  override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
-    if (excludedGroups.nonEmpty)
-      throw new UnsupportedOperationException("Programmer Error: group filtering not supported for single-tenant/in-memory workflow store")
-
+  override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
     val startableWorkflows = workflowStore filter { _._2 == WorkflowStoreState.Submitted } take n
     val updatedWorkflows = startableWorkflows map { _._1 -> WorkflowStoreState.Running }
     workflowStore = workflowStore ++ updatedWorkflows
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
index 389a18a009c..3513bd6b423 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
@@ -96,7 +96,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa
     * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp
     * flag to true
     */
-  override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
+  override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
     import cats.syntax.traverse._
     import common.validation.Validation._
     sqlDatabase.fetchWorkflowsInState(
@@ -106,8 +106,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa
       OffsetDateTime.now.toSystemTimestamp,
       WorkflowStoreState.Submitted.toString,
       WorkflowStoreState.Running.toString,
-      WorkflowStoreState.OnHold.toString,
-      excludedGroups: Set[String]
+      WorkflowStoreState.OnHold.toString
     ) map {
       // .get on purpose here to fail the future if something went wrong
       _.toList.traverse(fromWorkflowStoreEntry).toTry.get
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala
index 375a24db8b3..abc6cc71f75 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala
@@ -38,7 +38,7 @@ trait WorkflowStore {
     * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp
     * flag to true
     */
-  def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]]
+  def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)(implicit ec: ExecutionContext): Future[List[WorkflowToStart]]
 
   def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)],
                               heartbeatDateTime: OffsetDateTime)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala
index 2086cd34256..e44fef47e78 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala
@@ -27,7 +27,7 @@ sealed trait WorkflowStoreAccess {
                               heartbeatDateTime: OffsetDateTime)
                              (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int]
 
-  def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
+  def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
                              (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]]
 
   def abort(workflowId: WorkflowId)
@@ -50,9 +50,9 @@ case class UncoordinatedWorkflowStoreAccess(store: WorkflowStore) extends Workfl
     store.writeWorkflowHeartbeats(workflowIds.toVector.toSet, heartbeatDateTime)
   }
 
-  override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
+  override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
                                       (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] = {
-    store.fetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl, excludedGroups)
+    store.fetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl)
   }
 
   override def deleteFromStore(workflowId: WorkflowId)(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = {
@@ -79,9 +79,9 @@ case class CoordinatedWorkflowStoreAccess(coordinatedWorkflowStoreAccessActor: A
     )
   }
 
-  override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
+  override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
                                       (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] = {
-    val message = WorkflowStoreCoordinatedAccessActor.FetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl, excludedGroups)
+    val message = WorkflowStoreCoordinatedAccessActor.FetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl)
     withRetryForTransactionRollback(
       () => coordinatedWorkflowStoreAccessActor.ask(message).mapTo[List[WorkflowToStart]]
     )
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
index fe5a4d991da..9310cb36ad0 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
@@ -61,7 +61,7 @@ final case class WorkflowStoreActor private(
 
 object WorkflowStoreActor {
   sealed trait WorkflowStoreActorEngineCommand
-  final case class FetchRunnableWorkflows(n: Int, excludedGroups: Set[String]) extends WorkflowStoreActorEngineCommand
+  final case class FetchRunnableWorkflows(n: Int) extends WorkflowStoreActorEngineCommand
   final case class AbortWorkflowCommand(id: WorkflowId) extends WorkflowStoreActorEngineCommand
   final case class WorkflowOnHoldToSubmittedCommand(id: WorkflowId) extends WorkflowStoreActorEngineCommand
   case object InitializerCommand extends WorkflowStoreActorEngineCommand
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala
index 3d36a5a8062..c814cd09f5f 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala
@@ -33,8 +33,8 @@ class WorkflowStoreCoordinatedAccessActor(workflowStore: WorkflowStore) extends
   override def receive: Receive = {
     case WriteHeartbeats(ids, heartbeatDateTime) =>
       workflowStore.writeWorkflowHeartbeats(ids.toVector.toSet, heartbeatDateTime) |> run
-    case FetchStartableWorkflows(count, cromwellId, heartbeatTtl, excludedGroups) =>
-      workflowStore.fetchStartableWorkflows(count, cromwellId, heartbeatTtl, excludedGroups) |> run
+    case FetchStartableWorkflows(count, cromwellId, heartbeatTtl) =>
+      workflowStore.fetchStartableWorkflows(count, cromwellId, heartbeatTtl) |> run
     case DeleteFromStore(workflowId) =>
       workflowStore.deleteFromStore(workflowId) |> run
     case Abort(workflowId) =>
@@ -45,7 +45,7 @@ class WorkflowStoreCoordinatedAccessActor(workflowStore: WorkflowStore) extends
 object WorkflowStoreCoordinatedAccessActor {
   final case class WriteHeartbeats(workflowIds: NonEmptyVector[(WorkflowId, OffsetDateTime)],
                                    heartbeatDateTime: OffsetDateTime)
-  final case class FetchStartableWorkflows(count: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
+  final case class FetchStartableWorkflows(count: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
   final case class DeleteFromStore(workflowId: WorkflowId)
   final case class Abort(workflowId: WorkflowId)
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
index 5f066feeb76..ee699b78022 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
@@ -100,8 +100,8 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore,
 
   private def startNewWork(command: WorkflowStoreActorEngineCommand, sndr: ActorRef, nextData: WorkflowStoreActorData) = {
     val work: Future[Any] = command match {
-      case FetchRunnableWorkflows(count, excludedGroups) =>
-        newWorkflowMessage(count, excludedGroups) map { response =>
+      case FetchRunnableWorkflows(count) =>
+        newWorkflowMessage(count) map { response =>
           response match {
             case NewWorkflowsToStart(workflows) =>
               val workflowsIds = workflows.map(_.id).toList
@@ -185,10 +185,10 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore,
   /**
     * Fetches at most n workflows, and builds the correct response message based on if there were any workflows or not
     */
-  private def newWorkflowMessage(maxWorkflows: Int, excludedGroups: Set[String]): Future[WorkflowStoreEngineActorResponse] = {
+  private def newWorkflowMessage(maxWorkflows: Int): Future[WorkflowStoreEngineActorResponse] = {
     def fetchStartableWorkflowsIfNeeded = {
       if (maxWorkflows > 0) {
-        workflowStoreAccess.fetchStartableWorkflows(maxWorkflows, workflowHeartbeatConfig.cromwellId, workflowHeartbeatConfig.ttl, excludedGroups)
+        workflowStoreAccess.fetchStartableWorkflows(maxWorkflows, workflowHeartbeatConfig.cromwellId, workflowHeartbeatConfig.ttl)
       } else {
         Future.successful(List.empty[WorkflowToStart])
       }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
index 223d9ac0034..147f9a347cf 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
@@ -13,101 +13,29 @@ import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
+import org.scalatest.enablers.Emptiness._
 import org.scalatest.time.{Millis, Seconds, Span}
 import org.specs2.mock.Mockito
-import spray.json.{JsObject, JsString}
 
+import scala.concurrent.{ExecutionContext, Future}
 import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext, Future}
 
 class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalaFutures with BeforeAndAfterAll with Mockito {
   implicit val ec = ExecutionContext.global
-  implicit val defaultPatience = PatienceConfig(scaled(Span(20, Seconds)), scaled(Span(100, Millis)))
-
-  val onHoldSourceFilesCollection = NonEmptyList.of(
-    WorkflowSourceFilesCollection(
-      Option("sample"),
-      None,
-      None,
-      None,
-      None,
-      "input",
-      WorkflowOptions.empty,
-      "string",
-      None,
-      workflowOnHold = true,
-      Seq.empty,
-      None
-    )
-  )
-
-  val excludedGroupSourceFilesCollection = NonEmptyList.of(
-    WorkflowSourceFilesCollection(
-      Option("sample"),
-      None,
-      None,
-      None,
-      None,
-      "input",
-      WorkflowOptions(JsObject(Map("hogGroup" -> JsString("Zardoz")))),
-      "string",
-      None,
-      workflowOnHold = false,
-      Seq.empty,
-      None
-    )
-  )
-
-  val includedGroupSourceFilesCollection1 = NonEmptyList.of(
-    WorkflowSourceFilesCollection(
-      Option("sample"),
-      None,
-      None,
-      None,
-      None,
-      "input",
-      WorkflowOptions(JsObject(Map("hogGroup" -> JsString("Goldfinger")))),
-      "string",
-      None,
-      workflowOnHold = false,
-      Seq.empty,
-      None
-    )
-  )
-
-  val includedGroupSourceFilesCollection2 = NonEmptyList.of(
-    WorkflowSourceFilesCollection(
-      Option("sample"),
-      None,
-      None,
-      None,
-      None,
-      "input",
-      WorkflowOptions(JsObject(Map("hogGroup" -> JsString("Highlander")))),
-      "string",
-      None,
-      workflowOnHold = false,
-      Seq.empty,
-      None
-    )
-  )
-
-  val includedGroupSourceFilesCollection3 = NonEmptyList.of(
-    WorkflowSourceFilesCollection(
-      Option("sample"),
-      None,
-      None,
-      None,
-      None,
-      "input",
-      WorkflowOptions(JsObject(Map("hogGroup" -> JsString("Finding Forrester")))),
-      "string",
-      None,
-      workflowOnHold = false,
-      Seq.empty,
-      None
-    )
-  )
+  implicit val defaultPatience = PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(100, Millis)))
+  val sourceFilesCollection = NonEmptyList.of(WorkflowSourceFilesCollection(
+    Option("sample"),
+    None,
+    None,
+    None,
+    None,
+    "input",
+    WorkflowOptions.empty,
+    "string",
+    None,
+    workflowOnHold = true,
+    Seq.empty,
+    requestedWorkflowId = None))
 
   DatabaseSystem.All foreach { databaseSystem =>
 
@@ -120,29 +48,17 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     lazy val workflowStore = SqlWorkflowStore(dataAccess, metadataDataAccess)
 
-    def updateWfToRunning(startableWorkflows: List[WorkflowToStart]): Unit = {
-      startableWorkflows.foreach { wf =>
-        Await.result(workflowStore.sqlDatabase.updateWorkflowState(
-          wf.id.toString,
-          WorkflowStoreState.Submitted.toString,
-          WorkflowStoreState.Running.toString
-        ), 5.seconds)
-      }
-    }
-
     it should "start container if required" taggedAs DbmsTest in {
-      containerOpt.foreach {
-        _.start
-      }
+      containerOpt.foreach { _.start }
     }
 
     it should "honor the onHold flag" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second, Set.empty)
+        submissionResponses <- workflowStore.add(sourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         _ <- workflowStore.switchOnHoldToSubmitted(submissionResponses.head.id)
-        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second, Set.empty)
+        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second)
         _ = startableWorkflows2.map(_.id).intersect(submissionResponses.map(_.id).toList).size should be(1)
         _ <- workflowStore.deleteFromStore(startableWorkflows2.head.id) // Tidy up
       } yield ()).futureValue
@@ -150,8 +66,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold workflow" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A01", 1.second, Set.empty)
+        submissionResponses <- workflowStore.add(sourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A01", 1.second)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         workflowStoreAbortResponse <- workflowStore.abort(abortWorkflowId)
@@ -162,8 +78,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold then submitted workflow without a heartbeat" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A02", 1.second, Set.empty)
+        submissionResponses <- workflowStore.add(sourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A02", 1.second)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         _ <- workflowStore.switchOnHoldToSubmitted(abortWorkflowId)
@@ -175,8 +91,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold then submitted workflow with a heartbeat" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A03", 1.second, Set.empty)
+        submissionResponses <- workflowStore.add(sourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A03", 1.second)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         _ <- workflowStore.switchOnHoldToSubmitted(abortWorkflowId)
@@ -189,8 +105,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold then running workflow without a heartbeat" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A04", 1.second, Set.empty)
+        submissionResponses <- workflowStore.add(sourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A04", 1.second)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         _ <- workflowStore.switchOnHoldToSubmitted(abortWorkflowId)
@@ -208,8 +124,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold then running workflow with a heartbeat" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A05", 1.second, Set.empty)
+        submissionResponses <- workflowStore.add(sourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A05", 1.second)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         _ <- workflowStore.switchOnHoldToSubmitted(abortWorkflowId)
@@ -235,160 +151,17 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       } yield ()).futureValue
     }
 
-    it should "find a grouped workflow normally when not excluding" taggedAs DbmsTest in {
-      (for {
-        submissionResponses <- workflowStore.add(excludedGroupSourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A06", 1.second, excludedGroups = Set.empty)
-        _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList).size should be(1)
-        _ <- workflowStore.deleteFromStore(startableWorkflows.head.id) // Tidy up
-      } yield ()).futureValue
-    }
-
-    it should "honor the excludedGroups parameter for a target group" taggedAs DbmsTest in {
-      (for {
-        submissionResponses <- workflowStore.add(excludedGroupSourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A07", 1.second, excludedGroups = Set("Zardoz"))
-        _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
-        _ <- workflowStore.deleteFromStore(submissionResponses.head.id) // Tidy up
-      } yield ()).futureValue
-    }
-
-    it should "start workflows from hog group with lowest count of running workflows" taggedAs DbmsTest in {
-      // first submission of 50 workflows for hogGroup "Goldfinger"
-      val goldFingerWorkflowIds = (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection1), 5.seconds)).flatMap(_.map(_.id).toList)
-
-      // second submission of 50 workflows for hogGroup "Highlander"
-      val highlanderWorkflowIds = (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection2), 5.seconds)).flatMap(_.map(_.id).toList)
-
-      for (_ <- 1 to 10) yield {
-        (for {
-          // since both hog groups have 0 workflows running, the hog groups are sorted alphabetically and first one is picked
-          startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
-          _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
-          _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
-          _ = updateWfToRunning(startableWorkflows1)
-
-          startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
-          _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander")
-          _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
-          _ = updateWfToRunning(startableWorkflows2)
-        } yield ()).futureValue
-      }
-
-      // remove entries from WorkflowStore
-      (goldFingerWorkflowIds ++ highlanderWorkflowIds).foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
-    }
-
-    it should "respect excludedHogGroups and start workflows from hog group with lowest count of running workflows" taggedAs DbmsTest in {
-      (for {
-        // first submission of 10 workflows for hogGroup "Goldfinger"
-        goldFingerSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1))
-        goldFingerWorkflowIds = goldFingerSubmissions.flatMap(_.map(_.id).toList)
-
-        // second submission of 10 workflows for hogGroup "Zardoz"
-        zardozSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(excludedGroupSourceFilesCollection))
-        zardozWorkflowIds = zardozSubmissions.flatMap(_.map(_.id).toList)
-
-        startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
-        _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
-        _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows1)
-
-        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
-        _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Goldfinger")
-        _ = startableWorkflows2.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows2)
-
-        // there are 10 workflows from hog group "Zardoz" in the store, but since the group is excluded, 0 workflows are returned here
-        startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
-        _ = startableWorkflows3.size should be(0)
-
-        // hog group "Zardoz" has tokens to run workflows, hence don't exclude it
-        startableWorkflows4 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
-        _ = startableWorkflows4.map(_.hogGroup.value).toSet.head should be("Zardoz")
-        _ = startableWorkflows4.map(_.id).foreach(x => zardozWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows4)
-
-        startableWorkflows5 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
-        _ = startableWorkflows5.map(_.hogGroup.value).toSet.head should be("Zardoz")
-        _ = startableWorkflows5.map(_.id).foreach(x => zardozWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows5)
-
-        // remove entries from WorkflowStore
-        workflowsList = goldFingerWorkflowIds ++ zardozWorkflowIds
-        _ = workflowsList.foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
-      } yield()).futureValue
-    }
-
-    it should "start workflows from hog group with lowest count of running workflows for multiple hog groups" taggedAs DbmsTest in {
-      (for {
-        // first submission of 10 workflows for hogGroup "Goldfinger"
-        goldFingerSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1))
-        goldFingerWorkflowIds = goldFingerSubmissions.flatMap(_.map(_.id).toList)
-
-        // second submission of 10 workflows for hogGroup "Highlander"
-        highlanderSubmissions <- Future.sequence(for (_ <- 1 to 15) yield workflowStore.add(includedGroupSourceFilesCollection2))
-        highlanderWorkflowIds = highlanderSubmissions.flatMap(_.map(_.id).toList)
-
-        // since both hog groups have 0 workflows running, the hog groups are sorted alphabetically and first one is picked
-        startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
-        _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
-        _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows1)
-
-        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
-        _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander")
-        _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows2)
-
-        // new submission for hog group "Finding Forrester"
-        foresterSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection3))
-        foresterWorkflowIds = foresterSubmissions.flatMap(_.map(_.id).toList)
-
-        // now hog group "Finding Forrester" has 0 workflows running, hence it is picked to run
-        startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
-        _ = startableWorkflows3.map(_.hogGroup.value).toSet.head should be("Finding Forrester")
-        _ = startableWorkflows3.map(_.id).foreach(x => foresterWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows3)
-
-        // since all 3 hog groups have 5 workflows running each, the hog groups are sorted alphabetically and first one is picked
-        startableWorkflows4 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
-        _ = startableWorkflows4.map(_.hogGroup.value).toSet.head should be("Finding Forrester")
-        _ = startableWorkflows4.map(_.id).foreach(x => foresterWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows4)
-
-        startableWorkflows5 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
-        _ = startableWorkflows5.map(_.hogGroup.value).toSet.head should be("Goldfinger")
-        _ = startableWorkflows5.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows5)
-
-        startableWorkflows6 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
-        _ = startableWorkflows6.map(_.hogGroup.value).toSet.head should be("Highlander")
-        _ = startableWorkflows6.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows6)
-
-        startableWorkflows7 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
-        _ = startableWorkflows7.map(_.hogGroup.value).toSet.head should be("Highlander")
-        _ = startableWorkflows7.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
-        _ = updateWfToRunning(startableWorkflows7)
-
-        // remove entries from WorkflowStore
-        workflowsList = goldFingerWorkflowIds ++ highlanderWorkflowIds ++ foresterWorkflowIds
-        _ = workflowsList.foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
-      } yield ()).futureValue
-    }
-
     it should "accept and honor a requested workflow ID" taggedAs DbmsTest in {
       val requestedId = WorkflowId.randomId()
 
-      val sourcesToSubmit = onHoldSourceFilesCollection.map(c => c.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
+      val sourcesToSubmit = sourceFilesCollection.map(c => c.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
         requestedWorkflowId = Option(requestedId),
         workflowOnHold = false
       ))
 
       (for {
         submissionResponses <- workflowStore.add(sourcesToSubmit)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second, Set.empty)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList).size should be(1)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList).head should be(requestedId)
         _ <- workflowStore.deleteFromStore(requestedId) // tidy up
@@ -398,7 +171,7 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     it should "not accept a duplicate workflow ID" taggedAs DbmsTest in {
       val requestedId = WorkflowId.randomId()
 
-      val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
+      val workflowSourceFilesTemplate = sourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
         requestedWorkflowId = Option(requestedId)
       )
 
@@ -426,7 +199,7 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       val requestedId2 = WorkflowId.randomId()
       val requestedId3 = WorkflowId.randomId()
 
-      val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
+      val workflowSourceFilesTemplate = sourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
         requestedWorkflowId = Option(requestedId1)
       )
 
@@ -460,7 +233,7 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       val requestedId2 = WorkflowId.randomId()
       val requestedId3 = WorkflowId.randomId()
 
-      val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports]
+      val workflowSourceFilesTemplate = sourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports]
 
       val sourcesToSubmit = NonEmptyList.of(
         workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId1), workflowOnHold = false),
@@ -485,9 +258,7 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     }
 
     it should "stop container if required" taggedAs DbmsTest in {
-      containerOpt.foreach {
-        _.stop
-      }
+      containerOpt.foreach { _.stop }
     }
   }
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala
index b68a7f9e64b..6a12b978b53 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala
@@ -66,13 +66,13 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite
     val now = OffsetDateTime.now()
     val expected: List[WorkflowToStart] = List(WorkflowToStart(WorkflowId.randomId(), now, collection, Submitted, HogGroup("foo")))
     val workflowStore = new InMemoryWorkflowStore {
-      override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
+      override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
                                           (implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
         Future.successful(expected)
       }
     }
     val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore))
-    val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows success", 1.second, Set.empty)
+    val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows success", 1.second)
     implicit val timeout: Timeout = Timeout(2.seconds.dilated)
     actor.ask(request).mapTo[List[WorkflowToStart]] map { actual =>
       actual should be(expected)
@@ -97,13 +97,13 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite
     val now = OffsetDateTime.now()
     val expected: List[WorkflowToStart] = List(WorkflowToStart(WorkflowId.randomId(), now, collection, Submitted, HogGroup("foo")))
     val workflowStore = new InMemoryWorkflowStore {
-      override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
+      override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
                                           (implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
         Future.successful(expected)
       }
     }
     val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore))
-    val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows with workflow url success", 1.second, Set.empty)
+    val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows with workflow url success", 1.second)
     implicit val timeout: Timeout = Timeout(2.seconds.dilated)
     actor.ask(request).mapTo[List[WorkflowToStart]] map { actual =>
       actual should be(expected)
@@ -166,14 +166,14 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite
 
     it should s"fail to fetchStartableWorkflows due to $description" in {
       val workflowStore = new InMemoryWorkflowStore {
-        override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
+        override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
                                             (implicit ec: ExecutionContext): Future[Nothing] = {
           result()
         }
       }
       val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore))
       val heartbeatTtlNotReallyUsed = 1.second
-      val request = FetchStartableWorkflows(1, s"test $description fetchStartableWorkflows", heartbeatTtlNotReallyUsed, Set.empty)
+      val request = FetchStartableWorkflows(1, s"test $description fetchStartableWorkflows", heartbeatTtlNotReallyUsed)
       implicit val timeout: Timeout = Timeout(2.seconds.dilated)
       actor.ask(request).failed map { actual =>
         actual.getMessage should startWith(expectedMessagePrefix)
diff --git a/server/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/server/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
index da3021942ac..034d1429ab9 100644
--- a/server/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
+++ b/server/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
@@ -129,7 +129,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloCwlWorldSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
 
-      storeActor ! FetchRunnableWorkflows(2, Set.empty)
+      storeActor ! FetchRunnableWorkflows(2)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
           workflowNel.toList.size shouldBe 2
@@ -142,7 +142,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
           }
       }
 
-      storeActor ! FetchRunnableWorkflows(1, Set.empty)
+      storeActor ! FetchRunnableWorkflows(1)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
           workflowNel.toList.size shouldBe 1
@@ -182,7 +182,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(optionedSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
 
-      storeActor ! FetchRunnableWorkflows(1, Set.empty)
+      storeActor ! FetchRunnableWorkflows(1)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
           workflowNel.toList.size should be(1)
@@ -232,7 +232,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
 
-      storeActor ! FetchRunnableWorkflows(100, Set.empty)
+      storeActor ! FetchRunnableWorkflows(100)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
           workflowNel.toList.size shouldBe 3
@@ -260,7 +260,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
         "WorkflowStoreActor-RemainResponsiveForUnknown"
       )
 
-      storeActor ! FetchRunnableWorkflows(100, Set.empty)
+      storeActor ! FetchRunnableWorkflows(100)
       expectMsgPF(10 seconds) {
         case NoNewWorkflowsToStart => // Great
         case x => fail(s"Unexpected response from supposedly empty WorkflowStore: $x")
diff --git a/services/src/test/scala/cromwell/services/database/LobSpec.scala b/services/src/test/scala/cromwell/services/database/LobSpec.scala
index 042d901eab2..a92cf84aed9 100644
--- a/services/src/test/scala/cromwell/services/database/LobSpec.scala
+++ b/services/src/test/scala/cromwell/services/database/LobSpec.scala
@@ -122,7 +122,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = Option(Array.empty[Byte]).toBlobOption,
         customLabels = clob,
-        hogGroup = Option("abc-1"),
+        hogGroup = None,
       )
 
       val noneWorkflowUuid = WorkflowId.randomId().toString
@@ -141,7 +141,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = None,
         customLabels = clob,
-        hogGroup = Option("abc-1"),
+        hogGroup = None,
       )
 
       val aByte = 'a'.toByte
@@ -161,7 +161,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = Option(Array(aByte)).toBlobOption,
         customLabels = clob,
-        hogGroup = Option("abc-1"),
+        hogGroup = None,
       )
 
       val workflowStoreEntries = Seq(emptyWorkflowStoreEntry, noneWorkflowStoreEntry, aByteWorkflowStoreEntry)
@@ -175,8 +175,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
           heartbeatTimestampTo = OffsetDateTime.now.toSystemTimestamp,
           workflowStateFrom = "Submitted",
           workflowStateTo = "Running",
-          workflowStateExcluded = "On Hold",
-          excludedGroups = Set.empty
+          workflowStateExcluded = "On Hold"
         )
 
         _ = {

From a26e4f6a5960bce79f17a5778db93adbf0ea96a9 Mon Sep 17 00:00:00 2001
From: Brian Reilly <breilly@broadinstitute.org>
Date: Wed, 9 Mar 2022 14:20:42 -0500
Subject: [PATCH 09/19] Cromwell engine DRS checksum [BT-389] (#6683)

* introduce DrsHash

* reorder args

* moved and renamed DrsHash to FileHash
plus another baby step

* last baby step before digging into readAsString (and IO)

* baby step into restructuring readAsString

* changed to get hashes for all Path types, not just DrsPath
added a (currently failing) test
fixed an incorrect assertion in an existing test!

* added checksum checking for crc32c and md5
fixed test (still ugly)
removed custom retry in favor of existing IoActor retry

* added Enumeration of hash types

* refactor FileHash enum to HashType and add tests

* added test for success on retry
fixed issues with delayed IO evaluation on retry

* fix crc32c hash calculation
update hash failed message to aid debugging

* oops, forgot to run all NioFlow tests

* pushed getFileHash into DrsPath for easier mocking

* take IO out of DrsPath
add DrsPath NioFlow tests

* restricted Cromwell Engine checksums to DrsPath only

* distinguish between base64-encoded GCS crc32c and hex string (we hope) DRS crc32c

* follow-on to previous commit; forgot to update NioFlowSpec

* hoping to fix the trivy check failures

* add specific flavors of etag
decouple HashType (in cloud-nio-spi) from DrsCloudNioRegularFileAttributes (in cloud-nio-impl-drs)

* add ChecksumSkipped
---
 .github/workflows/trivy.yml                   |   2 +
 .../DrsCloudNioRegularFileAttributes.scala    |  35 +++---
 .../drs/DrsCloudNioFileProviderSpec.scala     |   5 +-
 .../nio/impl/drs/DrsPathResolverSpec.scala    |  25 ++--
 .../ftp/FtpCloudNioFileProviderSpec.scala     |   4 +-
 .../cloud/nio/spi/CloudFileAttributes.scala   |  12 +-
 .../main/scala/cloud/nio/spi/HashType.scala   |  50 ++++++++
 .../scala/cloud/nio/spi/HashTypeSpec.scala    |  41 +++++++
 .../engine/io/RetryableRequestSupport.scala   |   4 +-
 .../cromwell/engine/io/nio/NioFlow.scala      | 107 ++++++++++++------
 .../cromwell/engine/io/IoActorSpec.scala      |  22 ++--
 .../cromwell/engine/io/nio/NioFlowSpec.scala  |  71 ++++++++++--
 .../cromwell/filesystems/drs/DrsPath.scala    |  20 +++-
 13 files changed, 307 insertions(+), 91 deletions(-)
 create mode 100644 cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/HashType.scala
 create mode 100644 cloud-nio/cloud-nio-spi/src/test/scala/cloud/nio/spi/HashTypeSpec.scala

diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
index eb561d0cae6..0590d48ef53 100644
--- a/.github/workflows/trivy.yml
+++ b/.github/workflows/trivy.yml
@@ -21,6 +21,8 @@ jobs:
 
       # fetch SBT package
       - uses: olafurpg/setup-scala@v10
+        with:
+          java-version: adopt@1.11
 
       # set up SBT cache
       - uses: actions/cache@v2
diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala
index edd52c79b78..3c9293f2bd7 100644
--- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala
+++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala
@@ -2,14 +2,14 @@ package cloud.nio.impl.drs
 
 import java.nio.file.attribute.FileTime
 import java.time.{LocalDateTime, OffsetDateTime, ZoneOffset}
-
 import cats.effect.IO
-import cloud.nio.spi.CloudNioRegularFileAttributes
+import cloud.nio.spi.HashType.HashType
+import cloud.nio.spi.{CloudNioRegularFileAttributes, FileHash, HashType}
 import org.apache.commons.lang3.exception.ExceptionUtils
 
 class DrsCloudNioRegularFileAttributes(drsPath: String,
                                        sizeOption: Option[Long],
-                                       hashOption: Option[String],
+                                       hashOption: Option[FileHash],
                                        timeCreatedOption: Option[FileTime],
                                        timeUpdatedOption: Option[FileTime],
                                       ) extends CloudNioRegularFileAttributes{
@@ -18,7 +18,7 @@ class DrsCloudNioRegularFileAttributes(drsPath: String,
 
   override def size(): Long = sizeOption.getOrElse(0)
 
-  override def fileHash: Option[String] = hashOption
+  override def fileHash: Option[FileHash] = hashOption
 
   override def creationTime(): FileTime = timeCreatedOption.getOrElse(lastModifiedTime())
 
@@ -26,25 +26,20 @@ class DrsCloudNioRegularFileAttributes(drsPath: String,
 }
 
 object DrsCloudNioRegularFileAttributes {
-  private val priorityHashList: Seq[String] = Seq("crc32c", "md5", "sha256")
+  private val priorityHashList: Seq[(String, HashType)] = Seq(
+    ("crc32c", HashType.Crc32c),
+    ("md5", HashType.Md5),
+    ("sha256", HashType.Sha256),
+    ("etag", HashType.S3Etag)
+  )
 
-  def getPreferredHash(hashesOption: Option[Map[String, String]]): Option[String] = {
+  def getPreferredHash(hashesOption: Option[Map[String, String]]): Option[FileHash] = {
     hashesOption match {
-      case Some(hashes) if hashes.nonEmpty =>
-        val preferredHash = priorityHashList collectFirst {
-          case hashKey if hashes.contains(hashKey) => hashes(hashKey)
+      case Some(hashes: Map[String, String]) if hashes.nonEmpty =>
+        priorityHashList collectFirst {
+          case (key, hashType) if hashes.contains(key) => FileHash(hashType, hashes(key))
         }
-
-        // if no preferred hash was found, sort the hashes alphabetically by type and take the first one
-        Option(
-          preferredHash.getOrElse(
-            hashes.toSeq minBy {
-              case (hashType, _) => hashType
-            } match {
-              case (_, hashValue) => hashValue
-            }
-          )
-        )
+        // if no preferred hash was found, go ahead and return none because we don't support anything that the DRS object is offering
       case _ => None
     }
   }
diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala
index a6ecdfce8b0..aeb2ba42b0f 100644
--- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala
+++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala
@@ -3,6 +3,7 @@ package cloud.nio.impl.drs
 import cats.data.NonEmptyList
 import cats.effect.IO
 import cloud.nio.impl.drs.DrsCloudNioFileProvider.DrsReadInterpreter
+import cloud.nio.spi.{FileHash, HashType}
 import com.typesafe.config.ConfigFactory
 import common.assertion.CromwellTimeoutSpec
 import org.apache.commons.compress.utils.SeekableInMemoryByteChannel
@@ -140,7 +141,7 @@ class DrsCloudNioFileProviderSpec extends AnyFlatSpecLike with CromwellTimeoutSp
             size = Option(789L),
             timeCreated = Option(OffsetDateTime.ofInstant(instantCreated, ZoneOffset.UTC).toString),
             timeUpdated = Option(OffsetDateTime.ofInstant(instantUpdated, ZoneOffset.UTC).toString),
-            hashes = Option(Map("rot13" -> "gg0217869")),
+            hashes = Option(Map("md5" -> "gg0217869")),
           )
         )
       }
@@ -153,7 +154,7 @@ class DrsCloudNioFileProviderSpec extends AnyFlatSpecLike with CromwellTimeoutSp
     drsFileAttributes.creationTime().toMillis should be(123L)
     drsFileAttributes.lastModifiedTime().toMillis should be(456L)
     drsFileAttributes.size() should be(789L)
-    drsFileAttributes.fileHash should be(Option("gg0217869"))
+    drsFileAttributes.fileHash should be(Option(FileHash(HashType.Md5, "gg0217869")))
   }
 
   it should "throw exceptions for unsupported methods" in {
diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala
index 11285db9d72..78585c96ed9 100644
--- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala
+++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala
@@ -4,6 +4,7 @@ import java.nio.file.attribute.FileTime
 import java.time.OffsetDateTime
 
 import cloud.nio.impl.drs.DrsCloudNioRegularFileAttributes._
+import cloud.nio.spi.{FileHash, HashType}
 import common.assertion.CromwellTimeoutSpec
 import io.circe.{Json, JsonObject}
 import org.apache.http.ProtocolVersion
@@ -38,18 +39,19 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with
     fullMarthaResponse
       .copy(timeUpdated = fullMarthaResponse.timeUpdated.map(_.stripSuffix("Z") + "BADTZ"))
 
-  private val alfredHashValue = "xrd"
+  private val etagHashValue = "something"
   private val completeHashesMap = Option(Map(
     "betty" -> "abc123",
     "charles" -> "456",
-    "alfred" -> alfredHashValue,
+    "alfred" -> "xrd",
     "sha256" -> shaHashValue,
     "crc32c" -> crcHashValue,
     "md5" -> md5HashValue,
+    "etag" -> etagHashValue,
   ))
 
   private val missingCRCHashesMap = Option(Map(
-    "alfred" -> alfredHashValue,
+    "alfred" -> "xrd",
     "sha256" -> shaHashValue,
     "betty" -> "abc123",
     "md5" -> md5HashValue,
@@ -59,32 +61,33 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with
   private val onlySHAHashesMap = Option(Map(
     "betty" -> "abc123",
     "charles" -> "456",
-    "alfred" -> alfredHashValue,
+    "alfred" -> "xrd",
     "sha256" -> shaHashValue,
   ))
 
-  private val noPreferredHashesMap = Option(Map(
-    "alfred" -> alfredHashValue,
+  private val onlyEtagHashesMap = Option(Map(
+    "alfred" -> "xrd",
     "betty" -> "abc123",
     "charles" -> "456",
+    "etag" -> etagHashValue,
   ))
 
   behavior of "fileHash()"
 
   it should "return crc32c hash from `hashes` in Martha response when there is a crc32c" in {
-    DrsCloudNioRegularFileAttributes.getPreferredHash(completeHashesMap) shouldBe Option(crcHashValue)
+    DrsCloudNioRegularFileAttributes.getPreferredHash(completeHashesMap) shouldBe Option(FileHash(HashType.Crc32c, crcHashValue))
   }
 
   it should "return md5 hash from `hashes` in Martha response when there is no crc32c" in {
-    DrsCloudNioRegularFileAttributes.getPreferredHash(missingCRCHashesMap) shouldBe Option(md5HashValue)
+    DrsCloudNioRegularFileAttributes.getPreferredHash(missingCRCHashesMap) shouldBe Option(FileHash(HashType.Md5, md5HashValue))
   }
 
   it should "return sha256 hash from `hashes` in Martha response when there is only a sha256" in {
-    DrsCloudNioRegularFileAttributes.getPreferredHash(onlySHAHashesMap) shouldBe Option(shaHashValue)
+    DrsCloudNioRegularFileAttributes.getPreferredHash(onlySHAHashesMap) shouldBe Option(FileHash(HashType.Sha256, shaHashValue))
   }
 
-  it should "return first (alphabetized by type) hash from `hashes` in Martha response when there are no preferred hash types" in {
-    DrsCloudNioRegularFileAttributes.getPreferredHash(noPreferredHashesMap) shouldBe Option(alfredHashValue)
+  it should "return etag hash from `hashes` in Martha response when there is only an etag" in {
+    DrsCloudNioRegularFileAttributes.getPreferredHash(onlyEtagHashesMap) shouldBe Option(FileHash(HashType.S3Etag, etagHashValue))
   }
 
   it should "return None when no hashes object is returned" in {
diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala
index 47e4deca7b7..f68f67cbf67 100644
--- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala
+++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala
@@ -67,7 +67,7 @@ class FtpCloudNioFileProviderSpec extends AnyFlatSpec with CromwellTimeoutSpec w
       case _ => fail("Copy failed")
     }
   }
-  
+
   it should "throw when trying to copy files across hosts" in {
     an[UnsupportedOperationException] shouldBe thrownBy(fileProvider.copy("a", "b", "c", "d"))
   }
@@ -108,7 +108,7 @@ class FtpCloudNioFileProviderSpec extends AnyFlatSpec with CromwellTimeoutSpec w
     val writer = Channels.newOutputStream(writableByteChannel)
     writer.write("salut!".getBytes("ASCII"))
     writer.close()
-    
+
     Option(fakeUnixFileSystem.getEntry(file)) match {
       case Some(f: FileEntry) =>
         var content: String = ""
diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudFileAttributes.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudFileAttributes.scala
index 711d1c8efdf..189a835e8d0 100644
--- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudFileAttributes.scala
+++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudFileAttributes.scala
@@ -3,15 +3,23 @@ package cloud.nio.spi
 import java.nio.file.attribute.{BasicFileAttributes, FileTime}
 
 import cloud.nio.spi.CloudNioFileAttributes._
+import cloud.nio.spi.HashType.HashType
 
 trait CloudNioFileAttributes extends BasicFileAttributes {
-  def fileHash: Option[String]
+  def fileHash: Option[FileHash]
 }
 
 object CloudNioFileAttributes {
   val FileTimeZero: FileTime = FileTime.fromMillis(0)
 }
 
+case class FileHash(hashType: HashType, hash: String)
+
+sealed trait ChecksumResult
+case class ChecksumSkipped() extends ChecksumResult
+case class ChecksumSuccess() extends ChecksumResult
+case class ChecksumFailure(calculatedHash: String) extends ChecksumResult
+
 trait CloudNioRegularFileAttributes extends CloudNioFileAttributes {
   override def lastAccessTime(): FileTime = FileTimeZero
 
@@ -45,5 +53,5 @@ final case class CloudNioDirectoryAttributes(path: CloudNioPath) extends CloudNi
 
   override val fileKey: AnyRef = path
 
-  override val fileHash: Option[String] = None
+  override val fileHash: Option[FileHash] = None
 }
diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/HashType.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/HashType.scala
new file mode 100644
index 00000000000..87cbb9a9a83
--- /dev/null
+++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/HashType.scala
@@ -0,0 +1,50 @@
+package cloud.nio.spi
+
+import java.nio.{ByteBuffer, ByteOrder}
+import java.security.MessageDigest
+import java.util.Base64
+import java.util.zip.CRC32C
+
+object HashType extends Enumeration {
+  type HashType = Value
+
+  // crc32c as a hex string
+  val Crc32c: HashType.Value = Value
+  // GCS crc32c, which is base64-encoded instead of a hex string
+  val GcsCrc32c: HashType.Value = Value
+  val Md5: HashType.Value = Value
+  // Alibaba OSS etag
+  val OssEtag: HashType.Value = Value
+  // AWS S3 etag
+  val S3Etag: HashType.Value = Value
+  val Sha256: HashType.Value = Value
+
+  implicit class HashTypeValue(hashType: Value) {
+    def calculateHash(s: String): String = hashType match {
+      case Crc32c =>
+        val crc32c = new CRC32C()
+        crc32c.update(s.getBytes)
+        crc32c.getValue.toHexString
+      case GcsCrc32c =>
+        val crc32c = new CRC32C()
+        crc32c.update(s.getBytes)
+        val byteBuffer = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN)
+        byteBuffer.putInt(crc32c.getValue.toInt)
+        Base64.getEncoder.encodeToString(byteBuffer.array)
+      case Md5 =>
+        org.apache.commons.codec.digest.DigestUtils.md5Hex(s)
+      case OssEtag => throw new UnsupportedOperationException("Cannot currently calculate Alibaba OSS etag hash")
+      case S3Etag =>
+        val chunkSize = 8 * 1024 * 1024
+        val numChunks = (s.length.toDouble / chunkSize).ceil.toInt
+        val parts = s.getBytes.grouped(chunkSize).map(org.apache.commons.codec.digest.DigestUtils.md5Hex)
+        numChunks match {
+          case 1 => parts.next()
+          case _ =>
+            s"${org.apache.commons.codec.digest.DigestUtils.md5Hex(parts.mkString)}-${numChunks}"
+        }
+      case Sha256 =>
+        MessageDigest.getInstance("SHA-256").digest(s.getBytes).map("%02x" format _).mkString
+    }
+  }
+}
diff --git a/cloud-nio/cloud-nio-spi/src/test/scala/cloud/nio/spi/HashTypeSpec.scala b/cloud-nio/cloud-nio-spi/src/test/scala/cloud/nio/spi/HashTypeSpec.scala
new file mode 100644
index 00000000000..014a01c120a
--- /dev/null
+++ b/cloud-nio/cloud-nio-spi/src/test/scala/cloud/nio/spi/HashTypeSpec.scala
@@ -0,0 +1,41 @@
+package cloud.nio.spi
+
+import org.scalatest.flatspec.AnyFlatSpecLike
+import org.scalatest.matchers.should.Matchers
+
+class HashTypeSpec extends AnyFlatSpecLike with Matchers {
+
+  behavior of "HashType.calculateHash"
+
+  it should "calculate a crc32c hash" in {
+    HashType.Crc32c.calculateHash("hello") shouldBe "9a71bb4c"
+  }
+
+  it should "calculate a GCS crc32c hash" in {
+    HashType.GcsCrc32c.calculateHash("hello") shouldBe "mnG7TA=="
+  }
+
+  it should "calculate an etag hash on short data" in {
+    HashType.S3Etag.calculateHash("hello") shouldBe "5d41402abc4b2a76b9719d911017c592"
+  }
+
+  it should "calculate an etag hash on medium data" in {
+    val eightMB = 8 * 1024 * 1024
+    val value = Stream.continually(".").take(eightMB).mkString
+    HashType.S3Etag.calculateHash(value) shouldBe "f89801f68b5028d64e0238ffb5a1b8e0"
+  }
+
+  it should "calculate an etag hash on long data" in {
+    val eightMB = 8 * 1024 * 1024
+    val value = Stream.continually(".").take(eightMB + 1).mkString
+    HashType.S3Etag.calculateHash(value) shouldBe "8e224b463f4f5202c9621820f7690a01-2"
+  }
+
+  it should "calculate an md5 hash" in {
+    HashType.Md5.calculateHash("hello") shouldBe "5d41402abc4b2a76b9719d911017c592"
+  }
+
+  it should "calculate a sha256 hash" in {
+    HashType.Sha256.calculateHash("hello") shouldBe "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala b/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala
index 3dce13efe5f..03ef9224b7d 100644
--- a/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala
+++ b/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala
@@ -2,9 +2,10 @@ package cromwell.engine.io
 
 import java.io.IOException
 import java.net.{SocketException, SocketTimeoutException}
-
 import com.google.cloud.storage.StorageException
 import cromwell.engine.io.gcs.GcsBatchFlow.BatchFailedException
+import cromwell.engine.io.nio.ChecksumFailedException
+
 import javax.net.ssl.SSLException
 
 object RetryableRequestSupport {
@@ -21,6 +22,7 @@ object RetryableRequestSupport {
         AdditionalRetryableErrorMessages.contains(msg.toLowerCase))
     case _: SSLException => true
     case _: BatchFailedException => true
+    case _: ChecksumFailedException => true
     case _: SocketException => true
     case _: SocketTimeoutException => true
     case ioE: IOException if Option(ioE.getMessage).exists(_.contains("Error getting access token for service account")) => true
diff --git a/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala b/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala
index db1fd72903a..30a179995c7 100644
--- a/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala
+++ b/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala
@@ -2,7 +2,9 @@ package cromwell.engine.io.nio
 
 import akka.stream.scaladsl.Flow
 import cats.effect.{IO, Timer}
-import cloud.nio.impl.drs.DrsCloudNioFileSystemProvider
+
+import scala.util.Try
+import cloud.nio.spi.{ChecksumFailure, ChecksumResult, ChecksumSkipped, ChecksumSuccess, FileHash, HashType}
 import com.typesafe.config.Config
 import common.util.IORetry
 import cromwell.core.io._
@@ -60,7 +62,7 @@ class NioFlow(parallelism: Int,
       result <- operationResult
     } yield (result, commandContext)
 
-     io handleErrorWith {
+    io handleErrorWith {
       failure => IO.pure(commandContext.fail(failure))
     }
   }
@@ -105,12 +107,46 @@ class NioFlow(parallelism: Int,
     ()
   }
 
-  private def readAsString(read: IoContentAsStringCommand) = IO {
-    new String(
-      read.file.limitFileContent(read.options.maxBytes, read.options.failOnOverflow),
-      StandardCharsets.UTF_8
-    )
-      .replaceAll("\\r\\n", "\\\n")
+  private def readAsString(command: IoContentAsStringCommand): IO[String] = {
+    def checkHash(value: String, fileHash: FileHash): IO[ChecksumResult] = {
+      // Disable checksum validation if failOnOverflow is false. We might not read
+      // the entire stream, in which case the checksum will definitely fail.
+      // Ideally, we would only disable checksum validation if there was an actual
+      // overflow, but we don't know that here.
+      if (!command.options.failOnOverflow) return IO.pure(ChecksumSkipped())
+
+      val hash = fileHash.hashType.calculateHash(value)
+      if (hash.toLowerCase == fileHash.hash.toLowerCase) IO.pure(ChecksumSuccess())
+      else IO.pure(ChecksumFailure(hash))
+    }
+
+    def readFile: IO[String] = IO {
+      new String(
+        command.file.limitFileContent(command.options.maxBytes, command.options.failOnOverflow),
+        StandardCharsets.UTF_8
+      )
+    }
+
+    def readFileAndChecksum: IO[String] = {
+      for {
+        fileHash <- getHash(command.file)
+        uncheckedValue <- readFile
+        checksumResult <- checkHash(uncheckedValue, fileHash)
+        verifiedValue <- checksumResult match {
+          case _: ChecksumSkipped => IO.pure(uncheckedValue)
+          case _: ChecksumSuccess => IO.pure(uncheckedValue)
+          case failure: ChecksumFailure => IO.raiseError(
+            ChecksumFailedException(
+              s"Failed checksum for '${command.file}'. Expected '${fileHash.hashType}' hash of '${fileHash.hash}'. Calculated hash '${failure.calculatedHash}'"))
+        }
+      } yield verifiedValue
+    }
+
+    val fileContentIo = command.file match {
+      case _: DrsPath => readFileAndChecksum
+      case _ => readFile
+    }
+    fileContentIo.map(_.replaceAll("\\r\\n", "\\\n"))
   }
 
   private def size(size: IoSizeCommand) = IO {
@@ -118,17 +154,22 @@ class NioFlow(parallelism: Int,
   }
 
   private def hash(hash: IoHashCommand): IO[String] = {
-    hash.file match {
-      case gcsPath: GcsPath => IO.fromTry { gcsPath.objectBlobId.map(gcsPath.cloudStorage.get(_).getCrc32c) }
-      case drsPath: DrsPath => getFileHashForDrsPath(drsPath)
-      case s3Path: S3Path => IO { s3Path.eTag }
-      case ossPath: OssPath => IO { ossPath.eTag}
-      case path =>
-        IO.fromEither(
-        tryWithResource(() => path.newInputStream) { inputStream =>
-          org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream)
-        }.toEither
-      )
+    getHash(hash.file).map(_.hash)
+  }
+
+  private def getHash(file: Path): IO[FileHash] = {
+    file match {
+      case gcsPath: GcsPath => getFileHashForGcsPath(gcsPath)
+      case drsPath: DrsPath => IO {
+        drsPath.getFileHash
+      }
+      case s3Path: S3Path => IO {
+        FileHash(HashType.S3Etag, s3Path.eTag)
+      }
+      case ossPath: OssPath => IO {
+        FileHash(HashType.OssEtag, ossPath.eTag)
+      }
+      case path => getMd5FileHashForPath(path)
     }
   }
 
@@ -152,21 +193,21 @@ class NioFlow(parallelism: Int,
 
   private def createDirectories(path: Path) = path.parent.createDirectories()
 
-  private def getFileHashForDrsPath(drsPath: DrsPath): IO[String] = {
-    val drsFileSystemProvider = drsPath.drsPath.getFileSystem.provider.asInstanceOf[DrsCloudNioFileSystemProvider]
-
-    //Since this does not actually do any IO work it is not wrapped in IO
-    val fileAttributesOption = drsFileSystemProvider.fileProvider.fileAttributes(drsPath.drsPath.cloudHost, drsPath.drsPath.cloudPath)
+  /**
+    * Lazy evaluation of a Try in a delayed IO. This avoids accidentally eagerly evaluating the Try.
+    *
+    * IMPORTANT: Use this instead of IO.fromTry to make sure the Try will be reevaluated if the
+    * IoCommand is retried.
+    */
+  private def delayedIoFromTry[A](t: => Try[A]): IO[A] = IO[A] { t.get }
 
-    fileAttributesOption match {
-      case Some(fileAttributes) =>
-        val fileHashIO = IO { fileAttributes.fileHash }
+  private def getFileHashForGcsPath(gcsPath: GcsPath): IO[FileHash] = delayedIoFromTry {
+    gcsPath.objectBlobId.map(id => FileHash(HashType.GcsCrc32c, gcsPath.cloudStorage.get(id).getCrc32c))
+  }
 
-        fileHashIO.flatMap({
-          case Some(fileHash) => IO.pure(fileHash)
-          case None => IO.raiseError(new IOException(s"Error while resolving DRS path $drsPath. The response from Martha doesn't contain the 'md5' hash for the file."))
-        })
-      case None => IO.raiseError(new IOException(s"Error getting file hash of DRS path $drsPath. Reason: File attributes class DrsCloudNioRegularFileAttributes wasn't defined in DrsCloudNioFileProvider."))
+  private def getMd5FileHashForPath(path: Path): IO[FileHash] = delayedIoFromTry {
+    tryWithResource(() => path.newInputStream) { inputStream =>
+      FileHash(HashType.Md5, org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream))
     }
   }
 }
@@ -180,3 +221,5 @@ object NioFlow {
     NioFlowConfig(parallelism)
   }
 }
+
+case class ChecksumFailedException(message: String) extends IOException(message)
diff --git a/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala
index e3467227f0c..518788c26b0 100644
--- a/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala
@@ -12,6 +12,7 @@ import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.engine.io.IoActor.IoConfig
 import cromwell.engine.io.IoActorSpec.IoActorConfig
 import cromwell.engine.io.gcs.GcsBatchFlow.{BatchFailedException, GcsBatchFlowConfig}
+import cromwell.engine.io.nio.ChecksumFailedException
 import cromwell.engine.io.nio.NioFlow.NioFlowConfig
 import org.scalatest.flatspec.AnyFlatSpecLike
 import org.scalatest.matchers.should.Matchers
@@ -24,32 +25,32 @@ import scala.language.postfixOps
 
 class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender {
   behavior of "IoActor"
-  
+
   implicit val ec: ExecutionContext = system.dispatcher
   implicit val materializer: ActorMaterializer = ActorMaterializer()
-  
+
   override def afterAll(): Unit = {
     materializer.shutdown()
     super.afterAll()
   }
-  
+
   it should "copy a file" in {
     val testActor = TestActorRef(
       factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorCopy").ref, "cromwell test"),
       name = "testActorCopy",
     )
-    
+
     val src = DefaultPathBuilder.createTempFile()
     val dst: Path = src.parent.resolve(src.name + "-dst")
-    
+
     val copyCommand = DefaultIoCopyCommand(src, dst)
-    
+
     testActor ! copyCommand
     expectMsgPF(5 seconds) {
       case response: IoSuccess[_] => response.command.isInstanceOf[IoCopyCommand] shouldBe true
       case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
     }
-    
+
     dst.toFile should exist
     src.delete()
     dst.delete()
@@ -107,7 +108,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I
 
     testActor ! readCommand
     expectMsgPF(5 seconds) {
-      case response: IoSuccess[_] => 
+      case response: IoSuccess[_] =>
         response.command.isInstanceOf[IoContentAsStringCommand] shouldBe true
         response.result.asInstanceOf[String] shouldBe "hello"
       case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
@@ -253,6 +254,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I
       new StorageException(408, "message"),
       new StorageException(429, "message"),
       BatchFailedException(new Exception),
+      ChecksumFailedException("message"),
       new SocketException(),
       new SocketTimeoutException(),
       new IOException("text Error getting access token for service account some other text"),
@@ -273,7 +275,9 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I
       new IOException("Some other text. Could not read from gs://fc-secure-<snip>/JointGenotyping/<snip>/call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 504 Gateway Timeout"),
     )
 
-    retryables foreach { RetryableRequestSupport.isRetryable(_) shouldBe true }
+    retryables foreach { e => withClue(e) {
+      RetryableRequestSupport.isRetryable(e) shouldBe true }
+    }
   }
 
   it should "have correct non-retryable exceptions" in {
diff --git a/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala b/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala
index 161e0528c30..f4a583fefc8 100644
--- a/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala
@@ -4,6 +4,7 @@ import akka.actor.ActorRef
 import akka.stream.ActorMaterializer
 import akka.stream.scaladsl.{Keep, Sink, Source}
 import cats.effect.IO
+import cloud.nio.spi.{FileHash, HashType}
 import com.google.cloud.storage.StorageException
 import cromwell.core.io.DefaultIoCommandBuilder._
 import cromwell.core.io._
@@ -12,7 +13,9 @@ import cromwell.core.{CromwellFatalExceptionMarker, TestKitSuite}
 import cromwell.engine.io.IoActor.DefaultCommandContext
 import cromwell.engine.io.IoAttempts.EnhancedCromwellIoException
 import cromwell.engine.io.IoCommandContext
+import cromwell.filesystems.drs.DrsPath
 import cromwell.filesystems.gcs.GcsPath
+import org.mockito.Mockito.{times, verify, when}
 import org.scalatest.flatspec.AsyncFlatSpecLike
 import org.scalatest.matchers.should.Matchers
 import org.scalatestplus.mockito.MockitoSugar
@@ -20,6 +23,7 @@ import org.specs2.mock.Mockito._
 
 import java.nio.file.NoSuchFileException
 import java.util.UUID
+import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 import scala.language.postfixOps
 import scala.util.Failure
@@ -36,9 +40,9 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with
     parallelism = 1,
     onRetryCallback = NoopOnRetry,
     onBackpressure = NoopOnBackpressure,
-    numberOfAttempts = 5,
+    numberOfAttempts = 3,
     commandBackpressureStaleness = 5 seconds)(system.dispatcher).flow
-  
+
   implicit val materializer: ActorMaterializer = ActorMaterializer()
   private val replyTo = mock[ActorRef]
   private val readSink = Sink.head[(IoAck[_], IoCommandContext[_])]
@@ -63,15 +67,16 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with
   it should "read from a Nio Path" in {
     val testPath = DefaultPathBuilder.createTempFile()
     testPath.write("hello")
-    
-    val context = DefaultCommandContext(contentAsStringCommand(testPath, None, failOnOverflow = false).get, replyTo)
+
+    val context = DefaultCommandContext(contentAsStringCommand(testPath, None, failOnOverflow = true).get, replyTo)
     val testSource = Source.single(context)
 
     val stream = testSource.via(flow).toMat(readSink)(Keep.right)
-    
+
     stream.run() map {
       case (success: IoSuccess[_], _) => assert(success.result.asInstanceOf[String] == "hello")
-      case _ => fail("read returned an unexpected message")
+      case (ack, _) =>
+        fail(s"read returned an unexpected message:\n$ack\n\n")
     }
   }
 
@@ -89,7 +94,7 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with
       case _ => fail("size returned an unexpected message")
     }
   }
-  
+
   it should "get hash from a Nio Path" in {
     val testPath = DefaultPathBuilder.createTempFile()
     testPath.write("hello")
@@ -117,11 +122,55 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with
 
     stream.run() map {
       case (IoFailure(_, EnhancedCromwellIoException(_, receivedException)), _) =>
-        receivedException should be(receivedException)
+        receivedException should be(exception)
       case unexpected => fail(s"hash returned an unexpected message: $unexpected")
     }
   }
 
+  it should "fail if DrsPath hash doesn't match checksum" in {
+    val testPath = mock[DrsPath]
+    when(testPath.limitFileContent(any[Option[Int]], any[Boolean])(any[ExecutionContext])).thenReturn("hello".getBytes)
+    when(testPath.getFileHash).thenReturn(FileHash(HashType.Crc32c, "boom")) // correct Base64-encoded crc32c checksum is "9a71bb4c"
+
+    val context = DefaultCommandContext(contentAsStringCommand(testPath, Option(100), failOnOverflow = true).get, replyTo)
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (IoFailure(_, EnhancedCromwellIoException(_, receivedException)), _) =>
+        receivedException.getMessage should include ("Failed checksum")
+      case (ack, _) => fail(s"read returned an unexpected message:\n$ack\n\n")
+    }
+  }
+
+  it should "retry if DrsPath hash check fails, then succeed if the second check passes" in {
+    val testPath = mock[DrsPath]
+    when(testPath.limitFileContent(any[Option[Int]], any[Boolean])(any[ExecutionContext]))
+      .thenReturn("hola".getBytes)
+      .thenReturn("hello".getBytes)
+      .thenReturn("hello".getBytes)
+    when(testPath.getFileHash)
+      .thenReturn(FileHash(HashType.Crc32c, "9a71bb4c"))
+      .thenReturn(FileHash(HashType.Crc32c, "boom"))
+      .thenReturn(FileHash(HashType.Crc32c, "9a71bb4c"))
+
+    val context = DefaultCommandContext(contentAsStringCommand(testPath, Option(100), failOnOverflow = true).get, replyTo)
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map { result =>
+      verify(testPath, times(3)).limitFileContent(any[Option[Int]], any[Boolean])(any[ExecutionContext])
+      verify(testPath, times(3)).getFileHash
+      result match {
+        case (success: IoSuccess[_], _) => assert(success.result.asInstanceOf[String] == "hello")
+        case (ack, _) =>
+          fail(s"read returned an unexpected message:\n$ack\n\n")
+      }
+    }
+  }
+
   it should "copy Nio paths" in {
     val testPath = DefaultPathBuilder.createTempFile()
     val testCopyPath = testPath.sibling(UUID.randomUUID().toString)
@@ -141,10 +190,10 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with
   it should "copy Nio paths with" in {
     val testPath = DefaultPathBuilder.createTempFile()
     testPath.write("goodbye")
-    
+
     val testCopyPath = DefaultPathBuilder.createTempFile()
     testCopyPath.write("hello")
-    
+
     val context = DefaultCommandContext(copyCommand(testPath, testCopyPath).get, replyTo)
 
     val testSource = Source.single(context)
@@ -229,7 +278,7 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with
         }
        }
     }.flow
-    
+
     val stream = testSource.via(customFlow).toMat(readSink)(Keep.right)
 
     stream.run() map {
diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPath.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPath.scala
index 128ccc0b5a4..4f899c318bb 100644
--- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPath.scala
+++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPath.scala
@@ -1,8 +1,11 @@
 package cromwell.filesystems.drs
 
-import cloud.nio.spi.CloudNioPath
+import cloud.nio.impl.drs.DrsCloudNioFileSystemProvider
+import cloud.nio.spi.{CloudNioPath, FileHash}
 import cromwell.core.path.{NioPath, Path}
 
+import java.io.IOException
+
 
 case class DrsPath(drsPath: CloudNioPath, requesterPaysProjectIdOption: Option[String]) extends Path {
 
@@ -15,4 +18,19 @@ case class DrsPath(drsPath: CloudNioPath, requesterPaysProjectIdOption: Option[S
   override def pathAsString: String = drsPath.cloudHost
 
   override def pathWithoutScheme: String = pathAsString.stripPrefix(drsPath.getFileSystem.provider.getScheme + "://")
+
+  def getFileHash: FileHash = {
+    val drsFileSystemProvider = drsPath.getFileSystem.provider.asInstanceOf[DrsCloudNioFileSystemProvider]
+
+    val fileAttributesOption = drsFileSystemProvider.fileProvider.fileAttributes(drsPath.cloudHost, drsPath.cloudPath)
+
+    fileAttributesOption match {
+      case Some(fileAttributes) =>
+        fileAttributes.fileHash match {
+          case Some(fileHash) => fileHash
+          case None => throw new IOException(s"Error while resolving DRS path $this. The response from Martha doesn't contain the 'md5' hash for the file.")
+        }
+      case None => throw new IOException(s"Error getting file hash of DRS path $this. Reason: File attributes class DrsCloudNioRegularFileAttributes wasn't defined in DrsCloudNioFileProvider.")
+    }
+  }
 }

From d9187371fcb2319d3412d2c39973c9ff8d1bf30b Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@users.noreply.github.com>
Date: Wed, 9 Mar 2022 14:36:32 -0500
Subject: [PATCH 10/19] Shut down on heartbeat staleness [BT-589] (#6696)

---
 .../workflowstore/WorkflowStoreActor.scala    |  2 +-
 .../WorkflowStoreHeartbeatWriteActor.scala    | 68 ++++++++++++++-----
 2 files changed, 53 insertions(+), 17 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
index 9310cb36ad0..19aad6a1232 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
@@ -79,7 +79,7 @@ object WorkflowStoreActor {
 
   final case object GetWorkflowStoreStats
 
-  case class WorkflowStoreWriteHeartbeatCommand(workflowId: WorkflowId, submissionTime: OffsetDateTime)
+  case class WorkflowStoreWriteHeartbeatCommand(workflowId: WorkflowId, submissionTime: OffsetDateTime, heartbeatTime: OffsetDateTime = OffsetDateTime.now())
 
   def props(
              workflowStoreDatabase: WorkflowStore,
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActor.scala
index a74a44224c7..ee88627cdfe 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActor.scala
@@ -22,13 +22,13 @@ case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAc
                                             terminator: CromwellTerminator,
                                             override val serviceRegistryActor: ActorRef)
 
-  extends EnhancedBatchActor[(WorkflowId, OffsetDateTime)](
+  extends EnhancedBatchActor[WorkflowStoreWriteHeartbeatCommand](
     flushRate = workflowHeartbeatConfig.heartbeatInterval,
     batchSize = workflowHeartbeatConfig.writeBatchSize) {
 
   implicit val actorSystem: ActorSystem = context.system
 
-  override val threshold = workflowHeartbeatConfig.writeThreshold
+  override val threshold: Int = workflowHeartbeatConfig.writeThreshold
 
   private val failureShutdownDuration = workflowHeartbeatConfig.failureShutdownDuration
 
@@ -40,21 +40,57 @@ case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAc
     *
     * @return the number of elements processed
     */
-  override protected def process(data: NonEmptyVector[(WorkflowId, OffsetDateTime)]): Future[Int] = instrumentedProcess {
-    val heartbeatDateTime = OffsetDateTime.now()
-    val processFuture = workflowStoreAccess.writeWorkflowHeartbeats(data, heartbeatDateTime)
-    processFuture transform {
-      // Track the `Try`, and then return the original `Try`. Similar to `andThen` but doesn't swallow exceptions.
-      _ <| trackRepeatedFailures(heartbeatDateTime, data.length)
+  override protected def process(data: NonEmptyVector[WorkflowStoreWriteHeartbeatCommand]): Future[Int] = instrumentedProcess {
+    val now = OffsetDateTime.now()
+
+    val warnDuration = (failureShutdownDuration + workflowHeartbeatConfig.heartbeatInterval) / 2
+    val warnThreshold = warnDuration.toNanos
+    val errorThreshold = failureShutdownDuration.toNanos
+
+    // Traverse these heartbeats looking for staleness of warning or error severity.
+    val (warningIds, errorIds) = data.foldLeft((Seq.empty[WorkflowId], Seq.empty[WorkflowId])) { case ((w, e), h) =>
+      val staleness = JDuration.between(h.heartbeatTime, now).toNanos
+
+      // w = warning ids, e = error ids, h = heartbeat datum
+      if (staleness > errorThreshold) (w, e :+ h.workflowId)
+      else if (staleness > warnThreshold) (w :+ h.workflowId, e)
+      else (w, e)
+    }
+
+    if (warningIds.nonEmpty) {
+      log.warning(
+        "Found {} stale workflow heartbeats (more than {} old): {}",
+        warningIds.size.toString,
+        warnDuration.toString(),
+        warningIds.mkString(", ")
+      )
+    }
+
+    if (errorIds.isEmpty) {
+      val processFuture = workflowStoreAccess.writeWorkflowHeartbeats(data.map { h => (h.workflowId, h.submissionTime) }, now)
+      processFuture transform {
+        // Track the `Try`, and then return the original `Try`. Similar to `andThen` but doesn't swallow exceptions.
+        _ <| trackRepeatedFailures(now, data.length)
+      }
+    } else {
+      log.error(
+        "Shutting down Cromwell instance {} as {} stale workflow heartbeats (more than {} old) were found: {}",
+        workflowHeartbeatConfig.cromwellId,
+        errorIds.size.toString,
+        failureShutdownDuration.toString(),
+        errorIds.mkString(", ")
+      )
+      terminator.beginCromwellShutdown(WorkflowStoreHeartbeatWriteActor.Shutdown)
+      Future.successful(0)
     }
   }
 
-  override def receive = enhancedReceive.orElse(super.receive)
-  override protected def weightFunction(command: (WorkflowId, OffsetDateTime)) = 1
-  override protected def instrumentationPath = NonEmptyList.of("store", "heartbeat-writes")
-  override protected def instrumentationPrefix = InstrumentationPrefixes.WorkflowPrefix
-  override def commandToData(snd: ActorRef): PartialFunction[Any, (WorkflowId, OffsetDateTime)] = {
-    case command: WorkflowStoreWriteHeartbeatCommand => (command.workflowId, command.submissionTime)
+  override def receive: Receive = enhancedReceive.orElse(super.receive)
+  override protected def weightFunction(command: WorkflowStoreWriteHeartbeatCommand) = 1
+  override protected def instrumentationPath: NonEmptyList[String] = NonEmptyList.of("store", "heartbeat-writes")
+  override protected def instrumentationPrefix: Option[String] = InstrumentationPrefixes.WorkflowPrefix
+  override def commandToData(snd: ActorRef): PartialFunction[Any, WorkflowStoreWriteHeartbeatCommand] = {
+    case command: WorkflowStoreWriteHeartbeatCommand => command
   }
 
   /*
@@ -83,9 +119,9 @@ case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAc
           val failureUnits = failureShutdownDuration.unit
           val failureLength = FiniteDuration(failureJDuration.toNanos, TimeUnit.NANOSECONDS).toUnit(failureUnits)
           log.error(String.format(
-            "Shutting down %s as at least %s of heartbeat write errors have occurred between %s and %s (%s %s)",
+            "Shutting down %s as at least %d heartbeat write errors have occurred between %s and %s (%s %s)",
             workflowHeartbeatConfig.cromwellId,
-            failureShutdownDuration,
+            Integer.valueOf(workflowCount),
             lastSuccess,
             now,
             failureLength.toString,

From 2a7edc3d0e99df1b440aba4777cea69c4075f869 Mon Sep 17 00:00:00 2001
From: priyanka sebastian <sebastian.priyanka@gmail.com>
Date: Thu, 10 Mar 2022 06:09:55 -0800
Subject: [PATCH 11/19] created a new document HPCSlurmWithLocalScratch.md
 under docs/tutorials and created a link in HPCIntro.md [BT-592] (#6702)

* updated HPCIntro.md file with instructions to install cromwell to use local scratch device

* Create HPCSlurmWithLocalScratch.md

* reverting back to commit 5616d5b03b458c008271da0776cee7460b7acff7

* Added a link to HPCSlurmWithLocalScratch.md

Co-authored-by: root <root@broncos-clx03.jf.intel.com>
---
 docs/tutorials/HPCIntro.md                 |   1 +
 docs/tutorials/HPCSlurmWithLocalScratch.md | 210 +++++++++++++++++++++
 2 files changed, 211 insertions(+)
 create mode 100644 docs/tutorials/HPCSlurmWithLocalScratch.md

diff --git a/docs/tutorials/HPCIntro.md b/docs/tutorials/HPCIntro.md
index c57e62cd66f..bb08e0b2e65 100644
--- a/docs/tutorials/HPCIntro.md
+++ b/docs/tutorials/HPCIntro.md
@@ -289,3 +289,4 @@ You might find the following tutorials interesting to tackle next:
 
 * [Persisting Data Between Restarts](PersistentServer)
 * [Server Mode](ServerMode.md)
+* If you'd like to configure Cromwell to use a local scratch device, see instructions here [HPCSlurmWithLocalScratch.md](HPCSlurmWithLocalScratch.md)
diff --git a/docs/tutorials/HPCSlurmWithLocalScratch.md b/docs/tutorials/HPCSlurmWithLocalScratch.md
new file mode 100644
index 00000000000..39e85586557
--- /dev/null
+++ b/docs/tutorials/HPCSlurmWithLocalScratch.md
@@ -0,0 +1,210 @@
+####
+
+# Installing the Cromwell To Use Local Scratch Device
+#### These instructions are a community contribution
+
+In this section we will install the Cromwell Workflow Management system and configure it, so it can use the local scratch device on the compute nodes.
+(these installations are done in a ```centos 8``` enviornment)
+
+### 1. In order to install Cromwell, the `sbt` build tool is required, so we will install this now.
+
+##### a. Add the sbt online repository
+
+```hocon
+dnf config-manager --add-repo https://www.scala-sbt.org/sbt-rpm.repo
+```
+
+##### b. Install SBT build tool
+
+```hocon
+dnf -y install sbt
+```
+
+### 2. Now we can download Cromwell from git repository
+
+##### a. Create a directory into which we will install Cromwell
+
+```hocon
+mkdir cromwell
+```
+
+##### b. Now, we set the owner of that directory to the cromwell user and group and change the permissions in order to allow access to all users.
+
+```hocon
+chmod 775 -R cromwell
+chown -R cromwell:cromwell cromwell
+```
+
+##### c. Login to the cromwell user account.
+
+```hocon
+su - cromwell
+```
+
+##### d. We will now use git to clone the Cromwell git repository
+
+```hocon
+cd cromwell
+git clone https://github.com/broadinstitute/cromwell.git
+```
+
+##### e. This guide was tested and validated with `version 52` of cromwell, so we will checkout that version.
+
+```hocon
+cd cromwell
+git checkout 52
+```
+
+### 3. We will now configure Cromwell to use the local NVMe disk as scratch space
+
+##### a. Open the `backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala` file for editing
+
+##### b. Comment out line 3, so it reads:
+
+```hocon
+//import java.util.UUID
+```
+
+##### c. Remove lines 23 through 27, i.e. the following lines:
+
+```hocon
+val tempPath: String = {
+val uuid = UUID.randomUUID().toString
+val hash = uuid.substring(0, uuid.indexOf('-'))
+callRoot.resolve(s"tmp.$hash").pathAsString
+}
+```
+
+##### d. Add the following text in line 23 now:
+
+```hocon
+def tempPath: String = "/genomics_local"
+```
+
+##### e. Save and exit the file
+
+##### f. Open the file `backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala` for editing
+
+##### g. Go to line number 380. You will see the following content:
+
+```hocon
+|export _JAVA_OPTIONS=-Djava.io.tmpdir="$$tmpDir"
+|export TMPDIR="$$tmpDir"
+```
+
+
+##### h. Replace those two line with the following text:
+
+```hocon
+|mkdir -p $$tmpDir/tmp.$$$$
+|export _JAVA_OPTIONS=-Djava.io.tmpdir="$$tmpDir/tmp.$$$$"
+|export TMPDIR="$$tmpDir/tmp.$$$$
+```
+
+
+##### i. Save and exit the file.You can also execute the following commands:
+
+```hocon
+sed -i "s/^\(import\ java.util.UUID\)/\/\/\1/" \
+backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala
+sed -i '23,27d' \
+backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala
+sed -i '23i \ \ \ \ \ \ def tempPath: String = \"/genomics_local\"' \
+backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala
+sed 's/\(\s*|export _JAVA_OPTIONS.*\)\"/\ \ \ \ \ \ \ \ |mkdir -p \$\$tmpDir\/tmp\.\$\$\$\$\n\1\"/' \
+backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+sed 's/\(\s*|export _JAVA_OPTIONS.*tmpDir\)\"/\1\/tmp\.\$\$\$\$\"/' \
+backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+sed 's/\(\s*|export TMPDIR=.*tmpDir\)\"/\1\/tmp\.\$\$\$\$\"/' \
+backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+```
+
+### 4. Now we can build the patched Cromwell
+
+```hocon
+sbt clean
+sbt assembly
+```
+
+### 5. When the build was successful, we can move the new jar file into the cromwell directory
+
+```hocon
+cp server/target/scala-2.12/cromwell-52-*-SNAP.jar \
+cromwell/cromwell-52-fix.jar
+```
+
+## Configure the Execution Environment for Cromwell
+
+In this section we will configure the Cromwell execution environment.We will use the default configuration file as a starting point and change it to reflect our needs.
+*• Configure MariaDB as the database Cromwell should use
+*• Add the `SLURM backends`, so Cromwell can schedule job using `SLURM`
+
+### 1. First, we download the default Cromwell `reference.conf` configuration file.
+
+```hocon
+wget https://raw.githubusercontent.com/broadinstitute/cromwell/52_hotfix/core/\
+src/main/resources/reference.conf cromwell/
+```
+
+### 2. We will now add `SLURM` as the backend Cromwell should use
+
+##### a. Open the `cromwell/reference.conf` file for editing
+
+##### b. Goto line 479 which should read:
+
+```hocon
+default = "Local"
+```
+
+##### c. Change the `"Local"` in that line to `"SLURM"`
+
+##### d. Remove the following 5 lines, i.e. lines number 480 through 484
+
+```hocon
+providers {
+Local {
+actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+config {
+include required(classpath("reference_local_provider_config.inc.conf"))
+```
+
+##### e. Now add the following text after line 479, i.e. after the line reading `default ="SLURM"`. Ensure that the lines that show line-breaks in this document are, in fact, single lines in `reference.conf`
+
+```hocon
+providers {
+SLURM {
+#Modifying temp directory to write to local disk
+temporary-directory = "$(/genomics_local/)"
+actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+config {
+root = "cromwell-slurm-exec"
+runtime-attributes = """
+Int runtime_minutes = 600
+Int cpu = 2
+Int memory_mb = 1024
+String queue = "all"
+String? docker
+"""
+submit = """
+sbatch -J ${job_name} -D ${cwd} -o ${out} -e ${err} -t ${runtime_minutes} -p ${queue} ${"-c " +
+cpu} --mem ${memory_mb} --wrap "/bin/bash ${script}"
+"""
+kill = "scancel ${job_id}"
+check-alive = "squeue -j ${job_id}"
+job-id-regex = "Submitted batch job (\\d+).*"
+}
+}
+SLURM-BWA {
+temporary-directory = "$(/genomics_local/)"
+actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+config {
+root = "cromwell-slurm-exec"
+runtime-attributes = """
+Int runtime_minutes = 600
+Int cpu = 2
+Int memory_mb = 1024
+String queue = "bwa"
+String? docker
+"""
+submit = """
+```

From e408ad0a19a15d06acfb1ee62cde6a1776246906 Mon Sep 17 00:00:00 2001
From: Janet Gainer-Dewar <jdewar@broadinstitute.org>
Date: Thu, 10 Mar 2022 09:11:41 -0500
Subject: [PATCH 12/19] Convert int preemptible attrs to bool in TES (#6699)

---
 docs/backends/TES.md                          |  1 +
 .../impl/tes/TesRuntimeAttributes.scala       | 24 +++++++++++++++--
 .../impl/tes/TesRuntimeAttributesSpec.scala   | 27 +++++++++++++++++--
 3 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/docs/backends/TES.md b/docs/backends/TES.md
index 76923e4eced..da13116e893 100644
--- a/docs/backends/TES.md
+++ b/docs/backends/TES.md
@@ -63,6 +63,7 @@ This backend supports CPU, memory and disk size configuration through the use of
     * Type: String (ex: "1 GB" or "1024 MB")
 * `preemptible` defines whether or not to use preemptible VMs. 
     * Type: Boolean (ex: "true" or "false")
+    * Integers are accepted and will be converted to boolean (true if > 0)
 
 If they are not set, the TES backend may use default values.
 
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
index 103f8e51967..0895d7e1696 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
@@ -9,7 +9,7 @@ import eu.timepit.refined.api.Refined
 import eu.timepit.refined.numeric.Positive
 import wom.RuntimeAttributesKeys
 import wom.format.MemorySize
-import wom.types.WomStringType
+import wom.types.{WomIntegerType, WomStringType}
 import wom.values._
 
 case class TesRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode,
@@ -151,6 +151,26 @@ object PreemptibleValidation {
 class PreemptibleValidation extends BooleanRuntimeAttributesValidation(TesRuntimeAttributes.PreemptibleKey) {
   override def usedInCallCaching: Boolean = false
 
+  override protected def validateExpression: PartialFunction[WomValue, Boolean] = {
+    case womBoolValue if womType.coerceRawValue(womBoolValue).isSuccess => true
+    case womIntValue if WomIntegerType.coerceRawValue(womIntValue).isSuccess => true
+  }
+
+  override protected def validateValue: PartialFunction[WomValue, ErrorOr[Boolean]] = {
+    case value if womType.coerceRawValue(value).isSuccess =>
+      validateCoercedValue(womType.coerceRawValue(value).get.asInstanceOf[WomBoolean])
+    // The TES spec requires a boolean preemptible value, but many WDLs written originally
+    // for other backends use an integer. Interpret integers > 0 as true, others as false.
+    case value if WomIntegerType.coerceRawValue(value).isSuccess =>
+      validateCoercedValue(WomBoolean(WomIntegerType.coerceRawValue(value).get.asInstanceOf[WomInteger].value > 0))
+    case value if womType.coerceRawValue(value.valueString).isSuccess =>
+      /*
+      NOTE: This case statement handles WdlString("true") coercing to WdlBoolean(true).
+      For some reason "true" as String is coercable... but not the WdlString.
+       */
+      validateCoercedValue(womType.coerceRawValue(value.valueString).get.asInstanceOf[WomBoolean])
+  }
+
   override protected def missingValueMessage: String =
-    s"Expecting $key runtime attribute to be a Boolean or a String with values of 'true' or 'false'"
+    s"Expecting $key runtime attribute to be an Integer, Boolean, or a String with values of 'true' or 'false'"
 }
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
index 6314ed30d3a..9b3ee586ad9 100644
--- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
@@ -71,9 +71,32 @@ class TesRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeoutSpec
       assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
     }
 
-    "fail to validate an invalid preemptible entry" in {
+    "convert a positive integer preemptible entry to true boolean" in {
+      val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "preemptible" -> WomInteger(3))
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(preemptible = true)
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "convert a nonpositive integer preemptible entry to false boolean" in {
+      val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "preemptible" -> WomInteger(-1))
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(preemptible = false)
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "convert a valid string preemptible entry to boolean" in {
+      val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "preemptible" -> WomString("true"))
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(preemptible = true)
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "fail to validate an invalid string preemptible entry" in {
       val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "preemptible" -> WomString("yes"))
-      assertFailure(runtimeAttributes, "Expecting preemptible runtime attribute to be a Boolean or a String with values of 'true' or 'false'")
+      assertFailure(runtimeAttributes, "Expecting preemptible runtime attribute to be an Integer, Boolean, or a String with values of 'true' or 'false'")
+    }
+
+    "fail to validate an invalid type preemptible entry" in {
+      val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "preemptible" -> WomFloat(3.14))
+      assertFailure(runtimeAttributes, "Expecting preemptible runtime attribute to be an Integer, Boolean, or a String with values of 'true' or 'false'")
     }
 
     "validate a valid continueOnReturnCode entry" in {

From d2f2c9c123f253fe821de1f1623584cc5e12db05 Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Thu, 10 Mar 2022 17:59:40 -0500
Subject: [PATCH 13/19] @aednichols Option to exclude groups from workflow
 pickup [BW-1061][BW-1128] (#6704)

---
 .../slick/WorkflowStoreSlickDatabase.scala    |   5 +-
 .../tables/WorkflowStoreEntryComponent.scala  |  14 +-
 .../sql/WorkflowStoreSqlDatabase.scala        |   3 +-
 .../workflow/WorkflowManagerActor.scala       |   2 +-
 .../workflowstore/InMemoryWorkflowStore.scala |   6 +-
 .../workflowstore/SqlWorkflowStore.scala      |   5 +-
 .../workflowstore/WorkflowStore.scala         |   2 +-
 .../workflowstore/WorkflowStoreAccess.scala   |  10 +-
 .../workflowstore/WorkflowStoreActor.scala    |   2 +-
 .../WorkflowStoreCoordinatedAccessActor.scala |   6 +-
 .../WorkflowStoreEngineActor.scala            |   8 +-
 .../workflowstore/SqlWorkflowStoreSpec.scala  | 180 ++++++++++++++----
 ...kflowStoreCoordinatedAccessActorSpec.scala |  12 +-
 .../engine/WorkflowStoreActorSpec.scala       |  10 +-
 .../cromwell/services/database/LobSpec.scala  |   3 +-
 15 files changed, 194 insertions(+), 74 deletions(-)

diff --git a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
index 5d88d0dbec8..1ded96f6066 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
@@ -59,7 +59,8 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase {
                                      heartbeatTimestampTo: Timestamp,
                                      workflowStateFrom: String,
                                      workflowStateTo: String,
-                                     workflowStateExcluded: String)
+                                     workflowStateExcluded: String,
+                                     excludedGroups: Set[String])
                                     (implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]] = {
 
     def updateForFetched(cromwellId: String,
@@ -89,7 +90,7 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase {
 
     val action = for {
       workflowStoreEntries <- dataAccess.fetchStartableWorkflows(
-        (limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded)
+        limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded, excludedGroups
       ).result
       _ <- DBIO.sequence(
         workflowStoreEntries map updateForFetched(cromwellId, heartbeatTimestampTo, workflowStateFrom, workflowStateTo)
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index 3fb6ec48cdd..0b0bf2c510a 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -75,11 +75,11 @@ trait WorkflowStoreEntryComponent {
   /**
     * Returns up to "limit" startable workflows, sorted by submission time.
     */
-  val fetchStartableWorkflows = Compiled(
-    (limit: ConstColumn[Long],
-     heartbeatTimestampTimedOut: ConstColumn[Timestamp],
-     excludeWorkflowState: Rep[String]
-    ) => {
+  def fetchStartableWorkflows(limit: Long,
+     heartbeatTimestampTimedOut: Timestamp,
+     excludeWorkflowState: String,
+     excludedGroups: Set[String]
+    ): Query[WorkflowStoreEntries, WorkflowStoreEntry, Seq] = {
       val query = for {
         row <- workflowStoreEntries
         /*
@@ -93,11 +93,11 @@ trait WorkflowStoreEntryComponent {
         transaction that we know will impact those readers.
          */
         if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) &&
-          (row.workflowState =!= excludeWorkflowState)
+          (row.workflowState =!= excludeWorkflowState) &&
+          !(row.hogGroup inSet excludedGroups)
       } yield row
       query.forUpdate.sortBy(_.submissionTime.asc).take(limit)
     }
-  )
 
   /**
     * Useful for counting workflows in a given state.
diff --git a/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala
index a8312f6d685..82dc51c875b 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala
@@ -74,7 +74,8 @@ ____    __    ____  ______   .______       __  ___  _______  __        ______
                             heartbeatTimestampTo: Timestamp,
                             workflowStateFrom: String,
                             workflowStateTo: String,
-                            workflowStateExcluded: String)
+                            workflowStateExcluded: String,
+                            excludedGroups: Set[String])
                            (implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]]
 
   def writeWorkflowHeartbeats(workflowExecutionUuids: Seq[String],
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index b2bd7c53f1c..d51793a922b 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -173,7 +173,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
         Determine the number of available workflow slots and request the smaller of that number and maxWorkflowsToLaunch.
        */
       val maxNewWorkflows = maxWorkflowsToLaunch min (maxWorkflowsRunning - stateData.workflows.size - stateData.subWorkflows.size)
-      params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows)
+      params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows, excludedGroups = Set.empty) // Entry point for subsequent work in BW-675
       stay()
     case Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart, _) =>
       log.debug("WorkflowStore provided no new workflows to start")
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala
index 121371b162c..b6913f6172c 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala
@@ -1,7 +1,6 @@
 package cromwell.engine.workflow.workflowstore
 
 import java.time.OffsetDateTime
-
 import cats.data.NonEmptyList
 import cromwell.core.{HogGroup, WorkflowId, WorkflowSourceFilesCollection}
 import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreAbortResponse.WorkflowStoreAbortResponse
@@ -32,7 +31,10 @@ class InMemoryWorkflowStore extends WorkflowStore {
     * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp
     * flag to true
     */
-  override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
+  override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
+    if (excludedGroups.nonEmpty)
+      throw new UnsupportedOperationException("Programmer Error: group filtering not supported for single-tenant/in-memory workflow store")
+
     val startableWorkflows = workflowStore filter { _._2 == WorkflowStoreState.Submitted } take n
     val updatedWorkflows = startableWorkflows map { _._1 -> WorkflowStoreState.Running }
     workflowStore = workflowStore ++ updatedWorkflows
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
index 3513bd6b423..b36cb008933 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
@@ -96,7 +96,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa
     * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp
     * flag to true
     */
-  override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
+  override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
     import cats.syntax.traverse._
     import common.validation.Validation._
     sqlDatabase.fetchWorkflowsInState(
@@ -106,7 +106,8 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa
       OffsetDateTime.now.toSystemTimestamp,
       WorkflowStoreState.Submitted.toString,
       WorkflowStoreState.Running.toString,
-      WorkflowStoreState.OnHold.toString
+      WorkflowStoreState.OnHold.toString,
+      excludedGroups
     ) map {
       // .get on purpose here to fail the future if something went wrong
       _.toList.traverse(fromWorkflowStoreEntry).toTry.get
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala
index abc6cc71f75..375a24db8b3 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala
@@ -38,7 +38,7 @@ trait WorkflowStore {
     * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp
     * flag to true
     */
-  def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)(implicit ec: ExecutionContext): Future[List[WorkflowToStart]]
+  def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]]
 
   def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)],
                               heartbeatDateTime: OffsetDateTime)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala
index e44fef47e78..2086cd34256 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala
@@ -27,7 +27,7 @@ sealed trait WorkflowStoreAccess {
                               heartbeatDateTime: OffsetDateTime)
                              (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int]
 
-  def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
+  def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
                              (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]]
 
   def abort(workflowId: WorkflowId)
@@ -50,9 +50,9 @@ case class UncoordinatedWorkflowStoreAccess(store: WorkflowStore) extends Workfl
     store.writeWorkflowHeartbeats(workflowIds.toVector.toSet, heartbeatDateTime)
   }
 
-  override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
+  override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
                                       (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] = {
-    store.fetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl)
+    store.fetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl, excludedGroups)
   }
 
   override def deleteFromStore(workflowId: WorkflowId)(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = {
@@ -79,9 +79,9 @@ case class CoordinatedWorkflowStoreAccess(coordinatedWorkflowStoreAccessActor: A
     )
   }
 
-  override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
+  override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
                                       (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] = {
-    val message = WorkflowStoreCoordinatedAccessActor.FetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl)
+    val message = WorkflowStoreCoordinatedAccessActor.FetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl, excludedGroups)
     withRetryForTransactionRollback(
       () => coordinatedWorkflowStoreAccessActor.ask(message).mapTo[List[WorkflowToStart]]
     )
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
index 19aad6a1232..31a07bca87f 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
@@ -61,7 +61,7 @@ final case class WorkflowStoreActor private(
 
 object WorkflowStoreActor {
   sealed trait WorkflowStoreActorEngineCommand
-  final case class FetchRunnableWorkflows(n: Int) extends WorkflowStoreActorEngineCommand
+  final case class FetchRunnableWorkflows(n: Int, excludedGroups: Set[String]) extends WorkflowStoreActorEngineCommand
   final case class AbortWorkflowCommand(id: WorkflowId) extends WorkflowStoreActorEngineCommand
   final case class WorkflowOnHoldToSubmittedCommand(id: WorkflowId) extends WorkflowStoreActorEngineCommand
   case object InitializerCommand extends WorkflowStoreActorEngineCommand
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala
index c814cd09f5f..3d36a5a8062 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala
@@ -33,8 +33,8 @@ class WorkflowStoreCoordinatedAccessActor(workflowStore: WorkflowStore) extends
   override def receive: Receive = {
     case WriteHeartbeats(ids, heartbeatDateTime) =>
       workflowStore.writeWorkflowHeartbeats(ids.toVector.toSet, heartbeatDateTime) |> run
-    case FetchStartableWorkflows(count, cromwellId, heartbeatTtl) =>
-      workflowStore.fetchStartableWorkflows(count, cromwellId, heartbeatTtl) |> run
+    case FetchStartableWorkflows(count, cromwellId, heartbeatTtl, excludedGroups) =>
+      workflowStore.fetchStartableWorkflows(count, cromwellId, heartbeatTtl, excludedGroups) |> run
     case DeleteFromStore(workflowId) =>
       workflowStore.deleteFromStore(workflowId) |> run
     case Abort(workflowId) =>
@@ -45,7 +45,7 @@ class WorkflowStoreCoordinatedAccessActor(workflowStore: WorkflowStore) extends
 object WorkflowStoreCoordinatedAccessActor {
   final case class WriteHeartbeats(workflowIds: NonEmptyVector[(WorkflowId, OffsetDateTime)],
                                    heartbeatDateTime: OffsetDateTime)
-  final case class FetchStartableWorkflows(count: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
+  final case class FetchStartableWorkflows(count: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
   final case class DeleteFromStore(workflowId: WorkflowId)
   final case class Abort(workflowId: WorkflowId)
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
index ee699b78022..5f066feeb76 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
@@ -100,8 +100,8 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore,
 
   private def startNewWork(command: WorkflowStoreActorEngineCommand, sndr: ActorRef, nextData: WorkflowStoreActorData) = {
     val work: Future[Any] = command match {
-      case FetchRunnableWorkflows(count) =>
-        newWorkflowMessage(count) map { response =>
+      case FetchRunnableWorkflows(count, excludedGroups) =>
+        newWorkflowMessage(count, excludedGroups) map { response =>
           response match {
             case NewWorkflowsToStart(workflows) =>
               val workflowsIds = workflows.map(_.id).toList
@@ -185,10 +185,10 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore,
   /**
     * Fetches at most n workflows, and builds the correct response message based on if there were any workflows or not
     */
-  private def newWorkflowMessage(maxWorkflows: Int): Future[WorkflowStoreEngineActorResponse] = {
+  private def newWorkflowMessage(maxWorkflows: Int, excludedGroups: Set[String]): Future[WorkflowStoreEngineActorResponse] = {
     def fetchStartableWorkflowsIfNeeded = {
       if (maxWorkflows > 0) {
-        workflowStoreAccess.fetchStartableWorkflows(maxWorkflows, workflowHeartbeatConfig.cromwellId, workflowHeartbeatConfig.ttl)
+        workflowStoreAccess.fetchStartableWorkflows(maxWorkflows, workflowHeartbeatConfig.cromwellId, workflowHeartbeatConfig.ttl, excludedGroups)
       } else {
         Future.successful(List.empty[WorkflowToStart])
       }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
index 147f9a347cf..af49ef5149f 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
@@ -16,6 +16,7 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatest.enablers.Emptiness._
 import org.scalatest.time.{Millis, Seconds, Span}
 import org.specs2.mock.Mockito
+import spray.json.{JsObject, JsString}
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.concurrent.duration._
@@ -23,19 +24,91 @@ import scala.concurrent.duration._
 class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalaFutures with BeforeAndAfterAll with Mockito {
   implicit val ec = ExecutionContext.global
   implicit val defaultPatience = PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(100, Millis)))
-  val sourceFilesCollection = NonEmptyList.of(WorkflowSourceFilesCollection(
-    Option("sample"),
-    None,
-    None,
-    None,
-    None,
-    "input",
-    WorkflowOptions.empty,
-    "string",
-    None,
-    workflowOnHold = true,
-    Seq.empty,
-    requestedWorkflowId = None))
+
+  val onHoldSourceFilesCollection = NonEmptyList.of(
+    WorkflowSourceFilesCollection(
+      Option("sample"),
+      None,
+      None,
+      None,
+      None,
+      "input",
+      WorkflowOptions.empty,
+      "string",
+      None,
+      workflowOnHold = true,
+      Seq.empty,
+      None
+    )
+  )
+
+  val excludedGroupSourceFilesCollection = NonEmptyList.of(
+    WorkflowSourceFilesCollection(
+      Option("sample"),
+      None,
+      None,
+      None,
+      None,
+      "input",
+      WorkflowOptions(JsObject(Map("hogGroup" -> JsString("Zardoz")))),
+      "string",
+      None,
+      workflowOnHold = false,
+      Seq.empty,
+      None
+    )
+  )
+
+  val includedGroupSourceFilesCollection1 = NonEmptyList.of(
+    WorkflowSourceFilesCollection(
+      Option("sample"),
+      None,
+      None,
+      None,
+      None,
+      "input",
+      WorkflowOptions(JsObject(Map("hogGroup" -> JsString("Goldfinger")))),
+      "string",
+      None,
+      workflowOnHold = false,
+      Seq.empty,
+      None
+    )
+  )
+
+  val includedGroupSourceFilesCollection2 = NonEmptyList.of(
+    WorkflowSourceFilesCollection(
+      Option("sample"),
+      None,
+      None,
+      None,
+      None,
+      "input",
+      WorkflowOptions(JsObject(Map("hogGroup" -> JsString("Highlander")))),
+      "string",
+      None,
+      workflowOnHold = false,
+      Seq.empty,
+      None
+    )
+  )
+
+  val includedGroupSourceFilesCollection3 = NonEmptyList.of(
+    WorkflowSourceFilesCollection(
+      Option("sample"),
+      None,
+      None,
+      None,
+      None,
+      "input",
+      WorkflowOptions(JsObject(Map("hogGroup" -> JsString("Finding Forrester")))),
+      "string",
+      None,
+      workflowOnHold = false,
+      Seq.empty,
+      None
+    )
+  )
 
   DatabaseSystem.All foreach { databaseSystem =>
 
@@ -49,16 +122,18 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     lazy val workflowStore = SqlWorkflowStore(dataAccess, metadataDataAccess)
 
     it should "start container if required" taggedAs DbmsTest in {
-      containerOpt.foreach { _.start }
+      containerOpt.foreach {
+        _.start
+      }
     }
 
     it should "honor the onHold flag" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(sourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second)
+        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second, Set.empty)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         _ <- workflowStore.switchOnHoldToSubmitted(submissionResponses.head.id)
-        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second)
+        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second, Set.empty)
         _ = startableWorkflows2.map(_.id).intersect(submissionResponses.map(_.id).toList).size should be(1)
         _ <- workflowStore.deleteFromStore(startableWorkflows2.head.id) // Tidy up
       } yield ()).futureValue
@@ -66,8 +141,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold workflow" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(sourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A01", 1.second)
+        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A01", 1.second, Set.empty)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         workflowStoreAbortResponse <- workflowStore.abort(abortWorkflowId)
@@ -78,8 +153,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold then submitted workflow without a heartbeat" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(sourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A02", 1.second)
+        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A02", 1.second, Set.empty)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         _ <- workflowStore.switchOnHoldToSubmitted(abortWorkflowId)
@@ -91,8 +166,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold then submitted workflow with a heartbeat" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(sourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A03", 1.second)
+        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A03", 1.second, Set.empty)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         _ <- workflowStore.switchOnHoldToSubmitted(abortWorkflowId)
@@ -105,8 +180,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold then running workflow without a heartbeat" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(sourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A04", 1.second)
+        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A04", 1.second, Set.empty)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         _ <- workflowStore.switchOnHoldToSubmitted(abortWorkflowId)
@@ -124,8 +199,8 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     it should "abort an onHold then running workflow with a heartbeat" taggedAs DbmsTest in {
       (for {
-        submissionResponses <- workflowStore.add(sourceFilesCollection)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A05", 1.second)
+        submissionResponses <- workflowStore.add(onHoldSourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A05", 1.second, Set.empty)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
         abortWorkflowId = submissionResponses.head.id
         _ <- workflowStore.switchOnHoldToSubmitted(abortWorkflowId)
@@ -151,17 +226,54 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       } yield ()).futureValue
     }
 
+    it should "find a grouped workflow normally when not excluding" taggedAs DbmsTest in {
+      (for {
+        submissionResponses <- workflowStore.add(excludedGroupSourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A06", 1.second, excludedGroups = Set.empty)
+        _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList).size should be(1)
+        _ <- workflowStore.deleteFromStore(startableWorkflows.head.id) // Tidy up
+      } yield ()).futureValue
+    }
+
+    it should "honor the excludedGroups parameter for a target group" taggedAs DbmsTest in {
+      (for {
+        submissionResponses <- workflowStore.add(excludedGroupSourceFilesCollection)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A07", 1.second, excludedGroups = Set("Zardoz"))
+        _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList) should be(empty)
+        _ <- workflowStore.deleteFromStore(submissionResponses.head.id) // Tidy up
+      } yield ()).futureValue
+    }
+
+    it should "select appropriately with the excludedGroups parameter" taggedAs DbmsTest in {
+      (for {
+        submissionResponsesExcluded <- workflowStore.add(excludedGroupSourceFilesCollection)
+        submissionResponsesIncluded1 <- workflowStore.add(includedGroupSourceFilesCollection1)
+        submissionResponsesIncluded2 <- workflowStore.add(includedGroupSourceFilesCollection2)
+        submissionResponsesIncluded3 <- workflowStore.add(includedGroupSourceFilesCollection3)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(3, "A08", 1.second, excludedGroups = Set("Zardoz"))
+        _ = startableWorkflows.map(_.id).intersect(submissionResponsesExcluded.map(_.id).toList).size should be(0)
+        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded1.map(_.id).toList).size should be(1)
+        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded2.map(_.id).toList).size should be(1)
+        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded3.map(_.id).toList).size should be(1)
+        _ = startableWorkflows.map(_.id).size should be(3)
+        _ <- workflowStore.deleteFromStore(submissionResponsesExcluded.head.id) // Tidy up
+        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded1.head.id) // Tidy up
+        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded2.head.id) // Tidy up
+        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded3.head.id) // Tidy up
+      } yield ()).futureValue
+    }
+
     it should "accept and honor a requested workflow ID" taggedAs DbmsTest in {
       val requestedId = WorkflowId.randomId()
 
-      val sourcesToSubmit = sourceFilesCollection.map(c => c.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
+      val sourcesToSubmit = onHoldSourceFilesCollection.map(c => c.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
         requestedWorkflowId = Option(requestedId),
         workflowOnHold = false
       ))
 
       (for {
         submissionResponses <- workflowStore.add(sourcesToSubmit)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second)
+        startableWorkflows <- workflowStore.fetchStartableWorkflows(10, "A00", 1.second, Set.empty)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList).size should be(1)
         _ = startableWorkflows.map(_.id).intersect(submissionResponses.map(_.id).toList).head should be(requestedId)
         _ <- workflowStore.deleteFromStore(requestedId) // tidy up
@@ -171,7 +283,7 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     it should "not accept a duplicate workflow ID" taggedAs DbmsTest in {
       val requestedId = WorkflowId.randomId()
 
-      val workflowSourceFilesTemplate = sourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
+      val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
         requestedWorkflowId = Option(requestedId)
       )
 
@@ -199,7 +311,7 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       val requestedId2 = WorkflowId.randomId()
       val requestedId3 = WorkflowId.randomId()
 
-      val workflowSourceFilesTemplate = sourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
+      val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy(
         requestedWorkflowId = Option(requestedId1)
       )
 
@@ -233,7 +345,7 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       val requestedId2 = WorkflowId.randomId()
       val requestedId3 = WorkflowId.randomId()
 
-      val workflowSourceFilesTemplate = sourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports]
+      val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports]
 
       val sourcesToSubmit = NonEmptyList.of(
         workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId1), workflowOnHold = false),
@@ -258,7 +370,9 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
     }
 
     it should "stop container if required" taggedAs DbmsTest in {
-      containerOpt.foreach { _.stop }
+      containerOpt.foreach {
+        _.stop
+      }
     }
   }
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala
index 6a12b978b53..b68a7f9e64b 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala
@@ -66,13 +66,13 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite
     val now = OffsetDateTime.now()
     val expected: List[WorkflowToStart] = List(WorkflowToStart(WorkflowId.randomId(), now, collection, Submitted, HogGroup("foo")))
     val workflowStore = new InMemoryWorkflowStore {
-      override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
+      override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
                                           (implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
         Future.successful(expected)
       }
     }
     val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore))
-    val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows success", 1.second)
+    val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows success", 1.second, Set.empty)
     implicit val timeout: Timeout = Timeout(2.seconds.dilated)
     actor.ask(request).mapTo[List[WorkflowToStart]] map { actual =>
       actual should be(expected)
@@ -97,13 +97,13 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite
     val now = OffsetDateTime.now()
     val expected: List[WorkflowToStart] = List(WorkflowToStart(WorkflowId.randomId(), now, collection, Submitted, HogGroup("foo")))
     val workflowStore = new InMemoryWorkflowStore {
-      override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
+      override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
                                           (implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = {
         Future.successful(expected)
       }
     }
     val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore))
-    val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows with workflow url success", 1.second)
+    val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows with workflow url success", 1.second, Set.empty)
     implicit val timeout: Timeout = Timeout(2.seconds.dilated)
     actor.ask(request).mapTo[List[WorkflowToStart]] map { actual =>
       actual should be(expected)
@@ -166,14 +166,14 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite
 
     it should s"fail to fetchStartableWorkflows due to $description" in {
       val workflowStore = new InMemoryWorkflowStore {
-        override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration)
+        override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])
                                             (implicit ec: ExecutionContext): Future[Nothing] = {
           result()
         }
       }
       val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore))
       val heartbeatTtlNotReallyUsed = 1.second
-      val request = FetchStartableWorkflows(1, s"test $description fetchStartableWorkflows", heartbeatTtlNotReallyUsed)
+      val request = FetchStartableWorkflows(1, s"test $description fetchStartableWorkflows", heartbeatTtlNotReallyUsed, Set.empty)
       implicit val timeout: Timeout = Timeout(2.seconds.dilated)
       actor.ask(request).failed map { actual =>
         actual.getMessage should startWith(expectedMessagePrefix)
diff --git a/server/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/server/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
index 034d1429ab9..da3021942ac 100644
--- a/server/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
+++ b/server/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
@@ -129,7 +129,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloCwlWorldSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
 
-      storeActor ! FetchRunnableWorkflows(2)
+      storeActor ! FetchRunnableWorkflows(2, Set.empty)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
           workflowNel.toList.size shouldBe 2
@@ -142,7 +142,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
           }
       }
 
-      storeActor ! FetchRunnableWorkflows(1)
+      storeActor ! FetchRunnableWorkflows(1, Set.empty)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
           workflowNel.toList.size shouldBe 1
@@ -182,7 +182,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(optionedSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
 
-      storeActor ! FetchRunnableWorkflows(1)
+      storeActor ! FetchRunnableWorkflows(1, Set.empty)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
           workflowNel.toList.size should be(1)
@@ -232,7 +232,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
 
-      storeActor ! FetchRunnableWorkflows(100)
+      storeActor ! FetchRunnableWorkflows(100, Set.empty)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
           workflowNel.toList.size shouldBe 3
@@ -260,7 +260,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with CoordinatedWor
         "WorkflowStoreActor-RemainResponsiveForUnknown"
       )
 
-      storeActor ! FetchRunnableWorkflows(100)
+      storeActor ! FetchRunnableWorkflows(100, Set.empty)
       expectMsgPF(10 seconds) {
         case NoNewWorkflowsToStart => // Great
         case x => fail(s"Unexpected response from supposedly empty WorkflowStore: $x")
diff --git a/services/src/test/scala/cromwell/services/database/LobSpec.scala b/services/src/test/scala/cromwell/services/database/LobSpec.scala
index a92cf84aed9..74a3cc619d5 100644
--- a/services/src/test/scala/cromwell/services/database/LobSpec.scala
+++ b/services/src/test/scala/cromwell/services/database/LobSpec.scala
@@ -175,7 +175,8 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
           heartbeatTimestampTo = OffsetDateTime.now.toSystemTimestamp,
           workflowStateFrom = "Submitted",
           workflowStateTo = "Running",
-          workflowStateExcluded = "On Hold"
+          workflowStateExcluded = "On Hold",
+          excludedGroups = Set.empty
         )
 
         _ = {

From 879050402ae3ae769dfc6150ce0696d673a40dff Mon Sep 17 00:00:00 2001
From: Brian Reilly <breilly@broadinstitute.org>
Date: Mon, 14 Mar 2022 10:11:13 -0400
Subject: [PATCH 14/19] Scala-steward combined updates, part 1 [BT-560] (#6708)

Combines: #6628 #6629 #6630 #6631 #6635 #6636
---
 codegen_java/project/build.properties | 2 +-
 project/Dependencies.scala            | 6 +++---
 project/plugins.sbt                   | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/codegen_java/project/build.properties b/codegen_java/project/build.properties
index 10fd9eee04a..e64c208ff50 100644
--- a/codegen_java/project/build.properties
+++ b/codegen_java/project/build.properties
@@ -1 +1 @@
-sbt.version=1.5.5
+sbt.version=1.5.8
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 78b6f81bce2..93fadcf4155 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -101,7 +101,7 @@ object Dependencies {
   private val scalacheckV = "1.15.4"
   private val scalacticV = "3.2.10"
   private val scalameterV = "0.19"
-  private val scalamockV = "5.1.0"
+  private val scalamockV = "5.2.0"
   // scalatestV and flexmarkV must be upgraded together. Check the ScalaTest release notes to
   // find the version of FlexMark that corresponds to the new version of ScalaTest.
   private val scalatestV = "3.2.10"
@@ -121,8 +121,8 @@ object Dependencies {
     * Related Slick PR: https://github.com/slick/slick/pull/2101
   */
   private val slickV = "3.3.2-2076hotfix" // scala-steward:off (CROM-6620)
-  private val snakeyamlV = "1.29"
-  private val specs2MockV = "4.12.12"
+  private val snakeyamlV = "1.30"
+  private val specs2MockV = "4.13.3"
   private val sprayJsonV = "1.3.6"
   private val sttpV = "1.5.19" // scala-steward:off (CROM-6564)
   private val swaggerParserV = "1.0.56"
diff --git a/project/plugins.sbt b/project/plugins.sbt
index d498f9fac96..4d93de40a78 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,4 +1,4 @@
-addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.8.2")
+addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.9.0")
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0")
 addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.1")
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.0")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.3")

From a0dceca1d85b1e89f5a24adbaeb607d53d87fe42 Mon Sep 17 00:00:00 2001
From: Brian Reilly <breilly@broadinstitute.org>
Date: Tue, 15 Mar 2022 10:04:55 -0400
Subject: [PATCH 15/19] Scala-steward combined updates, part 2 [BT-560] (#6710)

* Combine scala-steward PRs: #6613 #6614 #6615 #6617 #6618 #6621 #6622 #6623 #6625 #6626
---
 project/Dependencies.scala | 18 +++++++++---------
 project/plugins.sbt        |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 93fadcf4155..e69411c60a4 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -8,7 +8,7 @@ object Dependencies {
   private val aliyunCoreV = "4.5.25"
   private val aliyunCrV = "4.1.2"
   private val aliyunOssV = "3.13.1"
-  private val ammoniteOpsV = "2.4.0"
+  private val ammoniteOpsV = "2.4.1"
   private val apacheHttpClientV = "4.5.13"
   private val awsSdkV = "2.17.50"
   // We would like to use the BOM to manage Azure SDK versions, but SBT doesn't support it.
@@ -61,7 +61,7 @@ object Dependencies {
   private val googleOauth2V = "1.1.0"
   private val googleOauthClientV = "1.32.1"
   private val googleCloudResourceManagerV = "1.1.2"
-  private val grpcV = "1.41.0"
+  private val grpcV = "1.45.0"
   private val guavaV = "31.0.1-jre"
   private val heterodonV = "1.0.0-beta3"
   private val hsqldbV = "2.6.0"
@@ -71,7 +71,7 @@ object Dependencies {
   private val jsr305V = "3.0.2"
   private val kindProjectorV = "0.10.0"
   private val kittensV = "2.3.2"
-  private val liquibaseV = "4.6.1"
+  private val liquibaseV = "4.8.0"
   private val logbackV = "1.2.10"
   private val lz4JavaV = "1.8.0"
   private val mariadbV = "2.7.4"
@@ -79,7 +79,7 @@ object Dependencies {
   The StatsD reporter for DropWizard's (Code Hale's) Metrics 3.x still works with Metrics 4.x.
   Still would be great to move to Prometheus / OpenCensus
    */
-  private val metrics4ScalaV = "4.1.19"
+  private val metrics4ScalaV = "4.2.8"
   private val metrics3StatsdV = "4.2.0"
   private val mockFtpServerV = "3.0.0"
   private val mockserverNettyV = "5.11.2"
@@ -89,10 +89,10 @@ object Dependencies {
   private val owlApiV = "5.1.19"
   private val paradiseV = "2.1.1"
   private val pegdownV = "1.6.0"
-  private val postgresV = "42.2.24"
-  private val pprintV = "0.6.6"
+  private val postgresV = "42.3.3"
+  private val pprintV = "0.7.1"
   private val rdf4jV = "3.7.1"
-  private val refinedV = "0.9.27"
+  private val refinedV = "0.9.28"
   private val rhinoV = "1.7.13"
   private val scalaCollectionCompatV = "2.5.0"
   private val scalaGraphV = "1.13.1"
@@ -128,10 +128,10 @@ object Dependencies {
   private val swaggerParserV = "1.0.56"
   private val swaggerUiV = "4.5.0"
   private val testContainersScalaV = "0.39.8"
-  private val tikaV = "2.1.0"
+  private val tikaV = "2.3.0"
   private val typesafeConfigV = "1.4.1"
   private val workbenchGoogleV = "0.21-5c9c4f6" // via: https://github.com/broadinstitute/workbench-libs/blob/develop/google/CHANGELOG.md
-  private val workbenchModelV = "0.14-89d0d9e" // via: https://github.com/broadinstitute/workbench-libs/blob/develop/model/CHANGELOG.md
+  private val workbenchModelV = "0.15-f9f0d4c" // via: https://github.com/broadinstitute/workbench-libs/blob/develop/model/CHANGELOG.md
   private val workbenchUtilV = "0.6-65bba14" // via: https://github.com/broadinstitute/workbench-libs/blob/develop/util/CHANGELOG.md
 
   private val slf4jFacadeDependencies = List(
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 4d93de40a78..db249558756 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,4 +1,4 @@
 addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.9.0")
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0")
-addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.1")
+addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.2")
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.3")

From 25849daa14727394f389f90f8a58f3a069d372be Mon Sep 17 00:00:00 2001
From: Brian Reilly <breilly@broadinstitute.org>
Date: Wed, 16 Mar 2022 12:50:49 -0400
Subject: [PATCH 16/19] Combine scala-steward PRs: #6606 #6607 #6608 #6609
 #6610 #6611 #6612 (#6711)

---
 project/Dependencies.scala | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index e69411c60a4..739bde41246 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -42,15 +42,15 @@ object Dependencies {
   // runtime dependency of ScalaTest. They must be upgraded together, based on the ScalaTest version.
   private val flexmarkV = "0.62.2" // scala-steward:off
   private val fs2V = "2.5.9" // scala-steward:off (CROM-6564)
-  private val googleApiClientV = "1.32.1"
-  private val googleCloudBigQueryV = "2.1.13"
-  // latest date via: https://mvnrepository.com/artifact/com.google.apis/google-api-services-cloudkms
-  private val googleCloudKmsV = "v1-rev20210820-1.32.1"
-  private val googleCloudMonitoringV = "3.0.6"
+  private val googleApiClientV = "1.33.2"
+  private val googleCloudBigQueryV = "2.10.0"
+  // latest date via: https://github.com/googleapis/google-api-java-client-services/blob/main/clients/google-api-services-cloudkms/v1.metadata.json
+  private val googleCloudKmsV = "v1-rev20220104-1.32.1"
+  private val googleCloudMonitoringV = "3.2.5"
   // BW-808 Pinning googleCloudNioV to this tried-and-true old version and quieting Scala Steward.
   // 0.121.2 is the most recent version currently known to work.
   private val googleCloudNioV = "0.61.0-alpha" // scala-steward:off
-  private val googleCloudStorageV = "2.1.6"
+  private val googleCloudStorageV = "2.1.10"
   private val googleGaxGrpcV = "2.4.1"
   // latest date via: https://mvnrepository.com/artifact/com.google.apis/google-api-services-genomics
   private val googleGenomicsServicesV2Alpha1ApiV = "v2alpha1-rev20210811-1.32.1"
@@ -58,9 +58,9 @@ object Dependencies {
   private val googleHttpClientV = "1.38.0"
   // latest date via: https://mvnrepository.com/artifact/com.google.apis/google-api-services-lifesciences
   private val googleLifeSciencesServicesV2BetaApiV = "v2beta-rev20210813-1.32.1"
-  private val googleOauth2V = "1.1.0"
-  private val googleOauthClientV = "1.32.1"
-  private val googleCloudResourceManagerV = "1.1.2"
+  private val googleOauth2V = "1.5.3"
+  private val googleOauthClientV = "1.33.1"
+  private val googleCloudResourceManagerV = "1.2.5"
   private val grpcV = "1.45.0"
   private val guavaV = "31.0.1-jre"
   private val heterodonV = "1.0.0-beta3"

From 677f7cab839b4d2a2896972d69a9b30ea5f5d11b Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Wed, 16 Mar 2022 19:01:03 -0400
Subject: [PATCH 17/19] Token exhaustion backpressures new workflow starts
 [BW-675] (#6692)

---
 .../workflow/WorkflowManagerActor.scala       | 12 +++++-
 .../tokens/JobTokenDispenserActor.scala       | 26 +++++++++++-
 .../workflow/tokens/TokenEventLogger.scala    | 40 ++++++++++++-------
 3 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index d51793a922b..4f929105bf0 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -16,6 +16,7 @@ import cromwell.engine.SubWorkflowStart
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.WorkflowActor._
 import cromwell.engine.workflow.WorkflowManagerActor._
+import cromwell.engine.workflow.tokens.JobTokenDispenserActor.{FetchLimitedGroups, ReplyLimitedGroups}
 import cromwell.engine.workflow.workflowstore.{WorkflowHeartbeatConfig, WorkflowStoreActor, WorkflowStoreEngineActor}
 import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSuccess, RegisterWorkflowCompleted}
 import cromwell.webservice.EngineStatsActor
@@ -167,13 +168,20 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
     /*
      Commands from clients
      */
-    case Event(RetrieveNewWorkflows, stateData) =>
+    case Event(RetrieveNewWorkflows, _) =>
       /*
         Cap the total number of workflows in flight, but also make sure we don't pull too many in at once.
         Determine the number of available workflow slots and request the smaller of that number and maxWorkflowsToLaunch.
        */
+      params.jobExecutionTokenDispenserActor ! FetchLimitedGroups
+      stay()
+    case Event(ReplyLimitedGroups(groups), stateData) =>
+      if (groups.nonEmpty)
+        log.info(s"Excluding groups from workflow launch: ${groups.mkString(", ")}")
+      else
+        log.debug("No groups excluded from workflow launch.")
       val maxNewWorkflows = maxWorkflowsToLaunch min (maxWorkflowsRunning - stateData.workflows.size - stateData.subWorkflows.size)
-      params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows, excludedGroups = Set.empty) // Entry point for subsequent work in BW-675
+      params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows, excludedGroups = groups)
       stay()
     case Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart, _) =>
       log.debug("WorkflowStore provided no new workflows to start")
diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActor.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActor.scala
index 8807ec2c69b..05bb9c8c94a 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActor.scala
@@ -64,7 +64,7 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef,
   lazy val effectiveLogInterval: Option[FiniteDuration] = logInterval.filterNot(_ == 0.seconds)
 
   lazy val tokenEventLogger = effectiveLogInterval match {
-    case Some(someInterval) => new CachingTokenEventLogger(log, someInterval)
+    case Some(someInterval: FiniteDuration) => new CachingTokenEventLogger(someInterval)
     case None => NullTokenEventLogger
   }
 
@@ -100,6 +100,7 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef,
       dispense(n)
     case Terminated(terminee) => onTerminate(terminee)
     case LogJobTokenAllocation(nextInterval) => logTokenAllocation(nextInterval)
+    case FetchLimitedGroups => sender ! tokenExhaustedGroups
     case ShutdownCommand => context stop self
   }
 
@@ -204,10 +205,31 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef,
 
     log.info(tokenDispenserState.asJson.printWith(Printer.spaces2))
 
+    tokenEventLogger.tokenExhaustedGroups foreach { group =>
+      log.info(s"Token Dispenser: The group $group has reached its job limit and is being rate-limited.")
+    }
+
+    tokenEventLogger.tokenExhaustedBackends foreach { backend =>
+      log.info(s"Token Dispenser: The backend $backend is starting too many jobs. New jobs are being limited.")
+    }
+
     // Schedule the next log event:
     context.system.scheduler.scheduleOnce(someInterval) { self ! LogJobTokenAllocation(someInterval) }(context.dispatcher)
     ()
   }
+
+  /*
+  This function is not backend-aware because:
+    (1) it is used for workflow pickup decisions, and the backend is not known at that time
+    (2) the modern purpose of tokens is to manage Cromwell capacity, not backend capacity,
+          so it's desirable that a group submitting to two or more backends pause workflow
+          pickup globally when it exhausts tokens in one of the backends
+   */
+  private def tokenExhaustedGroups: ReplyLimitedGroups = {
+    ReplyLimitedGroups(
+      tokenQueues.values.flatMap(_.eventLogger.tokenExhaustedGroups).toSet
+    )
+  }
 }
 
 object JobTokenDispenserActor {
@@ -222,6 +244,8 @@ object JobTokenDispenserActor {
   case object JobTokenReturn
   case object JobTokenDispensed
   final case class LogJobTokenAllocation(someInterval: FiniteDuration)
+  case object FetchLimitedGroups
+  final case class ReplyLimitedGroups(groups: Set[String])
 
   implicit val tokenEncoder = deriveEncoder[JobTokenType]
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenEventLogger.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenEventLogger.scala
index d585a03fe0c..27d1a10a421 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenEventLogger.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenEventLogger.scala
@@ -1,44 +1,54 @@
 package cromwell.engine.workflow.tokens
 
-import akka.event.LoggingAdapter
 import com.google.common.cache.CacheBuilder
 
 import scala.concurrent.duration.FiniteDuration
 
 trait TokenEventLogger {
   def flagTokenHog(hogGroup: String): Unit
+  def tokenExhaustedGroups: Set[String]
+
   def outOfTokens(backend: String): Unit
+  def tokenExhaustedBackends: Set[String]
 }
 
 case object NullTokenEventLogger extends TokenEventLogger {
   override def flagTokenHog(hogGroup: String): Unit = ()
+  override def tokenExhaustedGroups: Set[String] = Set.empty
+
   override def outOfTokens(backend: String): Unit = ()
+  override def tokenExhaustedBackends: Set[String] = Set.empty
 }
 
-class CachingTokenEventLogger(log: LoggingAdapter,
-                              cacheEntryTTL: FiniteDuration) extends TokenEventLogger {
-
-  log.info("Using CachingTokenEventLogger with an interval of {} to log token queue events", cacheEntryTTL)
+class CachingTokenEventLogger(cacheEntryTTL: FiniteDuration) extends TokenEventLogger {
 
-  private val cache = CacheBuilder.newBuilder()
+  private val groupCache = CacheBuilder.newBuilder()
     .expireAfterWrite(cacheEntryTTL._1, cacheEntryTTL._2)
     .maximumSize(10000)
     .build[String, Object]()
 
-
   override def flagTokenHog(hogGroup: String): Unit = {
-    if (Option(cache.getIfPresent("HOG_" + hogGroup)).isEmpty) {
-      log.info(s"Token Dispenser: The group $hogGroup has reached its job limit and is being rate-limited.")
-      cache.put("HOG_" + hogGroup, new Object())
-    }
+    groupCache.put(hogGroup, new Object())
+  }
+
+  override def tokenExhaustedGroups: Set[String] = {
+    import scala.collection.JavaConverters._
+    groupCache.asMap().keySet().asScala.toSet
   }
 
 
+  private val backendCache = CacheBuilder.newBuilder()
+    .expireAfterWrite(cacheEntryTTL._1, cacheEntryTTL._2)
+    .maximumSize(10000)
+    .build[String, Object]()
+
   override def outOfTokens(backend: String): Unit = {
-    if (Option(cache.getIfPresent("OOT_" + backend)).isEmpty) {
-      log.info(s"Token Dispenser: The backend $backend is starting too many jobs. New jobs are being limited.")
-      cache.put("OOT_" + backend, new Object())
-    }
+    backendCache.put(backend, new Object())
+  }
+
+  override def tokenExhaustedBackends: Set[String] = {
+    import scala.collection.JavaConverters._
+    backendCache.asMap().keySet().asScala.toSet
   }
 
 }

From fd097622b840deb0c00ca2eb233cbeaad7822f1a Mon Sep 17 00:00:00 2001
From: Saloni Shah <salonipshah11@gmail.com>
Date: Thu, 17 Mar 2022 12:07:01 -0400
Subject: [PATCH 18/19] [BW-1128] Pick up workflows code and flaky test fix
 (#6709)

---
 .../slick/WorkflowStoreSlickDatabase.scala    |  24 ++-
 .../tables/WorkflowStoreEntryComponent.scala  | 107 ++++++++++--
 .../workflowstore/SqlWorkflowStoreSpec.scala  | 153 +++++++++++++++---
 .../cromwell/services/database/LobSpec.scala  |   6 +-
 4 files changed, 246 insertions(+), 44 deletions(-)

diff --git a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
index 1ded96f6066..a39dcbf4195 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala
@@ -88,13 +88,25 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase {
       } yield ()
     }
 
+    def fetchAndUpdateStartableWfs(hogGroup: Option[String]) = {
+      for {
+        workflowStoreEntries <- dataAccess.fetchStartableWfsForHogGroup(
+          (limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded, hogGroup)
+        ).result
+        _ <- DBIO.sequence(
+          workflowStoreEntries map updateForFetched(cromwellId, heartbeatTimestampTo, workflowStateFrom, workflowStateTo)
+        )
+      } yield workflowStoreEntries
+    }
+
     val action = for {
-      workflowStoreEntries <- dataAccess.fetchStartableWorkflows(
-        limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded, excludedGroups
-      ).result
-      _ <- DBIO.sequence(
-        workflowStoreEntries map updateForFetched(cromwellId, heartbeatTimestampTo, workflowStateFrom, workflowStateTo)
-      )
+      // find hog group with lowest count of actively running workflows
+      hogGroupOption <- dataAccess.getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut, workflowStateExcluded, excludedGroups).result.headOption
+      workflowStoreEntries <- hogGroupOption match {
+        // if no such hog group was found, all hog groups have workflows that are either actively running or in "OnHold" status
+        case None => DBIO.successful(Seq.empty[WorkflowStoreEntry])
+        case Some(hogGroup) => fetchAndUpdateStartableWfs(hogGroup)
+      }
     } yield workflowStoreEntries
 
     // This should be safe because there are no repeated reads and the same rows are being locked for update as with
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index 0b0bf2c510a..0e29cee651f 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -73,31 +73,104 @@ trait WorkflowStoreEntryComponent {
   )
 
   /**
-    * Returns up to "limit" startable workflows, sorted by submission time.
-    */
-  def fetchStartableWorkflows(limit: Long,
-     heartbeatTimestampTimedOut: Timestamp,
-     excludeWorkflowState: String,
-     excludedGroups: Set[String]
-    ): Query[WorkflowStoreEntries, WorkflowStoreEntry, Seq] = {
-      val query = for {
-        row <- workflowStoreEntries
-        /*
+   * Return hog group with the lowest count of actively running workflows, that has nonzero startable workflows
+   */
+  def getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut: Timestamp,
+                                      excludeWorkflowState: String,
+                                      excludedGroups: Set[String]): Query[Rep[Option[String]], Option[String], Seq] = {
+    val startableWorkflows = for {
+      row <- workflowStoreEntries
+      /*
         This looks for:
-
         1) Workflows with no heartbeat (newly submitted or from a cleanly shut down Cromwell).
         2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell.
+        3) Workflows not in "OnHold" state
+        4) Workflows that don't belong to hog groups in excludedGroups
+      */
+      if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) &&
+        (row.workflowState =!= excludeWorkflowState) &&
+        !(row.hogGroup inSet excludedGroups)
+    } yield row
+
+    // calculates the count of startable workflows per hog group and oldest submission timestamp of workflow
+    // present in that hog group
+    val numOfStartableWfsByHogGroup: Query[
+      (Rep[Option[String]], Rep[Int], Rep[Option[Timestamp]]),
+      (Option[String], Int, Option[Timestamp]),
+      Seq
+    ] = startableWorkflows
+      .groupBy(_.hogGroup)
+      .map { case (hogGroupName, workflows) => (hogGroupName, workflows.length, workflows.map(_.submissionTime).min) }
+      .sortBy(_._2.asc)
+
+    val totalWorkflows = for {
+      row <- workflowStoreEntries
+      /*
+        This looks for:
+        1) Workflows not in "OnHold" state
+        2) Workflows that don't belong to hog groups in excludedGroups
+      */
+      if row.workflowState =!= excludeWorkflowState &&
+        !(row.hogGroup inSet excludedGroups)
+    } yield row
+
+    // calculates the count of total workflows per hog group
+    val totalWorkflowsByHogGroup: Query[(Rep[Option[String]], Rep[Int]), (Option[String], Int), Seq] = totalWorkflows
+      .groupBy(_.hogGroup)
+      .map { case (hogGroupName, workflows) => (hogGroupName, workflows.length) }
+      .sortBy(_._2.asc)
+
+    // calculates the number of actively running workflows for each hog group. If a hog group
+    // has all it's workflows that are either actively running or in "OnHold" status it is not
+    // included in the list. Hog groups that have no workflows actively running return count as 0
+    val wfsRunningPerHogGroup: Query[
+      (Rep[Option[String]], Rep[Int], Rep[Option[Timestamp]]),
+      (Option[String], Int, Option[Timestamp]),
+      Seq
+    ] = for {
+      (hog_group, workflows_ct) <- totalWorkflowsByHogGroup
+      (startable_hog_group, startable_workflows_ct, oldest_submission_time) <- numOfStartableWfsByHogGroup if hog_group === startable_hog_group
+    } yield (hog_group, workflows_ct - startable_workflows_ct, oldest_submission_time)
+
+    // sort the above calculated result set first by the count of actively running workflows, then by hog group with
+    // oldest submission timestamp and then sort it alphabetically by hog group name. Then take the first row of
+    // the result and return the hog group name.
+    wfsRunningPerHogGroup.sortBy {
+      case (hogGroupName, running_wf_ct, oldest_submission_time) => (running_wf_ct.asc, oldest_submission_time, hogGroupName)
+    }.take(1).map(_._1)
+  }
 
-        Workflows are taken by submission time, oldest first. This is a "query for update", meaning rows are
-        locked such that readers are blocked since we will do an update subsequent to this select in the same
-        transaction that we know will impact those readers.
-         */
+  /**
+   * Returns up to "limit" startable workflows, sorted by submission time, that belong to
+   * given hog group and are not in "OnHold" status.
+   */
+  val fetchStartableWfsForHogGroup = Compiled(
+    (limit: ConstColumn[Long],
+     heartbeatTimestampTimedOut: ConstColumn[Timestamp],
+     excludeWorkflowState: Rep[String],
+     hogGroup: Rep[Option[String]]) => {
+
+      val workflowsToStart = for {
+        row <- workflowStoreEntries
+        /*
+          This looks for:
+          1) Workflows with no heartbeat (newly submitted or from a cleanly shut down Cromwell).
+          2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell.
+          3) Workflows not in "OnHold" state
+          4) Workflows that belong to included hog group
+        */
         if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) &&
           (row.workflowState =!= excludeWorkflowState) &&
-          !(row.hogGroup inSet excludedGroups)
+          (row.hogGroup === hogGroup)
       } yield row
-      query.forUpdate.sortBy(_.submissionTime.asc).take(limit)
+
+      /*
+        This is a "query for update", meaning rows are locked such that readers are blocked since we will
+        do an update subsequent to this select in the same transaction that we know will impact those readers.
+       */
+      workflowsToStart.forUpdate.sortBy(_.submissionTime.asc).take(limit)
     }
+  )
 
   /**
     * Useful for counting workflows in a given state.
diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
index af49ef5149f..4433046fe4a 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
@@ -13,17 +13,16 @@ import org.scalatest.BeforeAndAfterAll
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
-import org.scalatest.enablers.Emptiness._
 import org.scalatest.time.{Millis, Seconds, Span}
 import org.specs2.mock.Mockito
 import spray.json.{JsObject, JsString}
 
-import scala.concurrent.{ExecutionContext, Future}
 import scala.concurrent.duration._
+import scala.concurrent.{Await, ExecutionContext, Future}
 
 class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalaFutures with BeforeAndAfterAll with Mockito {
   implicit val ec = ExecutionContext.global
-  implicit val defaultPatience = PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(100, Millis)))
+  implicit val defaultPatience = PatienceConfig(scaled(Span(20, Seconds)), scaled(Span(100, Millis)))
 
   val onHoldSourceFilesCollection = NonEmptyList.of(
     WorkflowSourceFilesCollection(
@@ -121,6 +120,16 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
 
     lazy val workflowStore = SqlWorkflowStore(dataAccess, metadataDataAccess)
 
+    def updateWfToRunning(startableWorkflows: List[WorkflowToStart]): Unit = {
+      startableWorkflows.foreach { wf =>
+        Await.result(workflowStore.sqlDatabase.updateWorkflowState(
+          wf.id.toString,
+          WorkflowStoreState.Submitted.toString,
+          WorkflowStoreState.Running.toString
+        ), 5.seconds)
+      }
+    }
+
     it should "start container if required" taggedAs DbmsTest in {
       containerOpt.foreach {
         _.start
@@ -244,22 +253,130 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat
       } yield ()).futureValue
     }
 
-    it should "select appropriately with the excludedGroups parameter" taggedAs DbmsTest in {
+    it should "start workflows from hog group with lowest count of running workflows" taggedAs DbmsTest in {
+      // first submission of 50 workflows for hogGroup "Goldfinger"
+      val goldFingerWorkflowIds = (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection1), 5.seconds)).flatMap(_.map(_.id).toList)
+
+      // second submission of 50 workflows for hogGroup "Highlander"
+      val highlanderWorkflowIds = (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection2), 5.seconds)).flatMap(_.map(_.id).toList)
+
+      for (_ <- 1 to 10) yield {
+        (for {
+          // since both hog groups have 0 workflows running, the hog group with oldest submission time is picked first
+          startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
+          _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+          _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+          _ = updateWfToRunning(startableWorkflows1)
+
+          startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
+          _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander")
+          _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
+          _ = updateWfToRunning(startableWorkflows2)
+        } yield ()).futureValue
+      }
+
+      // remove entries from WorkflowStore
+      (goldFingerWorkflowIds ++ highlanderWorkflowIds).foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
+    }
+
+    it should "respect excludedHogGroups and start workflows from hog group with lowest count of running workflows" taggedAs DbmsTest in {
+      (for {
+        // first submission of 10 workflows for hogGroup "Goldfinger"
+        goldFingerSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1))
+        goldFingerWorkflowIds = goldFingerSubmissions.flatMap(_.map(_.id).toList)
+
+        // second submission of 10 workflows for hogGroup "Zardoz"
+        zardozSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(excludedGroupSourceFilesCollection))
+        zardozWorkflowIds = zardozSubmissions.flatMap(_.map(_.id).toList)
+
+        startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
+        _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+        _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows1)
+
+        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
+        _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+        _ = startableWorkflows2.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows2)
+
+        // there are 10 workflows from hog group "Zardoz" in the store, but since the group is excluded, 0 workflows are returned here
+        startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz"))
+        _ = startableWorkflows3.size should be(0)
+
+        // hog group "Zardoz" has tokens to run workflows, hence don't exclude it
+        startableWorkflows4 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
+        _ = startableWorkflows4.map(_.hogGroup.value).toSet.head should be("Zardoz")
+        _ = startableWorkflows4.map(_.id).foreach(x => zardozWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows4)
+
+        startableWorkflows5 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String])
+        _ = startableWorkflows5.map(_.hogGroup.value).toSet.head should be("Zardoz")
+        _ = startableWorkflows5.map(_.id).foreach(x => zardozWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows5)
+
+        // remove entries from WorkflowStore
+        workflowsList = goldFingerWorkflowIds ++ zardozWorkflowIds
+        _ = workflowsList.foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
+      } yield()).futureValue
+    }
+
+    it should "start workflows from hog group with lowest count of running workflows for multiple hog groups" taggedAs DbmsTest in {
       (for {
-        submissionResponsesExcluded <- workflowStore.add(excludedGroupSourceFilesCollection)
-        submissionResponsesIncluded1 <- workflowStore.add(includedGroupSourceFilesCollection1)
-        submissionResponsesIncluded2 <- workflowStore.add(includedGroupSourceFilesCollection2)
-        submissionResponsesIncluded3 <- workflowStore.add(includedGroupSourceFilesCollection3)
-        startableWorkflows <- workflowStore.fetchStartableWorkflows(3, "A08", 1.second, excludedGroups = Set("Zardoz"))
-        _ = startableWorkflows.map(_.id).intersect(submissionResponsesExcluded.map(_.id).toList).size should be(0)
-        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded1.map(_.id).toList).size should be(1)
-        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded2.map(_.id).toList).size should be(1)
-        _ = startableWorkflows.map(_.id).intersect(submissionResponsesIncluded3.map(_.id).toList).size should be(1)
-        _ = startableWorkflows.map(_.id).size should be(3)
-        _ <- workflowStore.deleteFromStore(submissionResponsesExcluded.head.id) // Tidy up
-        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded1.head.id) // Tidy up
-        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded2.head.id) // Tidy up
-        _ <- workflowStore.deleteFromStore(submissionResponsesIncluded3.head.id) // Tidy up
+        // first submission of 10 workflows for hogGroup "Goldfinger"
+        goldFingerSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1))
+        goldFingerWorkflowIds = goldFingerSubmissions.flatMap(_.map(_.id).toList)
+
+        // second submission of 10 workflows for hogGroup "Highlander"
+        highlanderSubmissions <- Future.sequence(for (_ <- 1 to 15) yield workflowStore.add(includedGroupSourceFilesCollection2))
+        highlanderWorkflowIds = highlanderSubmissions.flatMap(_.map(_.id).toList)
+
+        // since both hog groups have 0 workflows running, the hog group with oldest submission time is picked first
+        startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+        _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows1)
+
+        startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander")
+        _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows2)
+
+        // new submission for hog group "Finding Forrester"
+        foresterSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection3))
+        foresterWorkflowIds = foresterSubmissions.flatMap(_.map(_.id).toList)
+
+        // now hog group "Finding Forrester" has 0 workflows running, hence it is picked to run
+        startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows3.map(_.hogGroup.value).toSet.head should be("Finding Forrester")
+        _ = startableWorkflows3.map(_.id).foreach(x => foresterWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows3)
+
+        // since all 3 hog groups have 5 workflows running each, the hog group with oldest submission time is picked first
+        startableWorkflows5 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows5.map(_.hogGroup.value).toSet.head should be("Goldfinger")
+        _ = startableWorkflows5.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows5)
+
+        // since both "Highlander" and "Finding Forrester" have 5 workflows in Running state, the hog group with oldest submission time is picked first
+        startableWorkflows6 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows6.map(_.hogGroup.value).toSet.head should be("Highlander")
+        _ = startableWorkflows6.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows6)
+
+        // "Finding Forrester" is now the hog group with least running workflows and has 5 more workflows to run, hence it is picked to run
+        startableWorkflows4 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows4.map(_.hogGroup.value).toSet.head should be("Finding Forrester")
+        _ = startableWorkflows4.map(_.id).foreach(x => foresterWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows4)
+
+        startableWorkflows7 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String])
+        _ = startableWorkflows7.map(_.hogGroup.value).toSet.head should be("Highlander")
+        _ = startableWorkflows7.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x))
+        _ = updateWfToRunning(startableWorkflows7)
+
+        // remove entries from WorkflowStore
+        workflowsList = goldFingerWorkflowIds ++ highlanderWorkflowIds ++ foresterWorkflowIds
+        _ = workflowsList.foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds))
       } yield ()).futureValue
     }
 
diff --git a/services/src/test/scala/cromwell/services/database/LobSpec.scala b/services/src/test/scala/cromwell/services/database/LobSpec.scala
index 74a3cc619d5..042d901eab2 100644
--- a/services/src/test/scala/cromwell/services/database/LobSpec.scala
+++ b/services/src/test/scala/cromwell/services/database/LobSpec.scala
@@ -122,7 +122,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = Option(Array.empty[Byte]).toBlobOption,
         customLabels = clob,
-        hogGroup = None,
+        hogGroup = Option("abc-1"),
       )
 
       val noneWorkflowUuid = WorkflowId.randomId().toString
@@ -141,7 +141,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = None,
         customLabels = clob,
-        hogGroup = None,
+        hogGroup = Option("abc-1"),
       )
 
       val aByte = 'a'.toByte
@@ -161,7 +161,7 @@ class LobSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Sc
         submissionTime = OffsetDateTime.now.toSystemTimestamp,
         importsZip = Option(Array(aByte)).toBlobOption,
         customLabels = clob,
-        hogGroup = None,
+        hogGroup = Option("abc-1"),
       )
 
       val workflowStoreEntries = Seq(emptyWorkflowStoreEntry, noneWorkflowStoreEntry, aByteWorkflowStoreEntry)

From 7152bc14fbfdea4c995724edb1aed426bda38bcf Mon Sep 17 00:00:00 2001
From: Brian Reilly <breilly@broadinstitute.org>
Date: Thu, 17 Mar 2022 17:06:05 -0400
Subject: [PATCH 19/19] Scala-steward combined updates, part 4 [BT-560] (#6714)

* Combine scala-steward PRs: #6602 #6603 #6605 #6616 #6624
---
 project/Dependencies.scala | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 739bde41246..9cdd3f73cb1 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -1,7 +1,7 @@
 import sbt._
 
 object Dependencies {
-  private val akkaHttpCirceIntegrationV = "1.38.2"
+  private val akkaHttpCirceIntegrationV = "1.39.2"
   private val akkaHttpV = "10.1.15" // (CROM-6619)
   private val akkaV = "2.5.32" // scala-steward:off (CROM-6637)
   private val aliyunBcsV = "6.2.4"
@@ -15,7 +15,7 @@ object Dependencies {
   // https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/boms/azure-sdk-bom
   // https://github.com/sbt/sbt/issues/4531
   private val azureIdentitySdkV = "1.4.2"
-  private val azureKeyVaultSdkV = "4.3.4"
+  private val azureKeyVaultSdkV = "4.3.7"
   private val betterFilesV = "3.9.1"
   /*
   cats-effect, fs2, http4s, and sttp (also to v3) should all be upgraded at the same time to use cats-effect 3.x.
@@ -51,7 +51,7 @@ object Dependencies {
   // 0.121.2 is the most recent version currently known to work.
   private val googleCloudNioV = "0.61.0-alpha" // scala-steward:off
   private val googleCloudStorageV = "2.1.10"
-  private val googleGaxGrpcV = "2.4.1"
+  private val googleGaxGrpcV = "2.12.2"
   // latest date via: https://mvnrepository.com/artifact/com.google.apis/google-api-services-genomics
   private val googleGenomicsServicesV2Alpha1ApiV = "v2alpha1-rev20210811-1.32.1"
   private val googleHttpClientApacheV = "2.1.2"
@@ -64,7 +64,7 @@ object Dependencies {
   private val grpcV = "1.45.0"
   private val guavaV = "31.0.1-jre"
   private val heterodonV = "1.0.0-beta3"
-  private val hsqldbV = "2.6.0"
+  private val hsqldbV = "2.6.1"
   private val http4sV = "0.21.31" // this release is EOL. We need to upgrade further for cats3. https://http4s.org/versions/
   private val jacksonV = "2.13.0"
   private val janinoV = "3.1.6"
@@ -127,7 +127,7 @@ object Dependencies {
   private val sttpV = "1.5.19" // scala-steward:off (CROM-6564)
   private val swaggerParserV = "1.0.56"
   private val swaggerUiV = "4.5.0"
-  private val testContainersScalaV = "0.39.8"
+  private val testContainersScalaV = "0.40.2"
   private val tikaV = "2.3.0"
   private val typesafeConfigV = "1.4.1"
   private val workbenchGoogleV = "0.21-5c9c4f6" // via: https://github.com/broadinstitute/workbench-libs/blob/develop/google/CHANGELOG.md