WX-1252 Runtime attributes cleanup – CWL runtime environment (#7369)

backend/src/main/scala/cromwell/backend/Command.scala

@@ -3,7 +3,6 @@ package cromwell.backend
 import common.validation.ErrorOr._
 import common.validation.Validation._
 import wom.InstantiatedCommand
-import wom.callable.RuntimeEnvironment
 import wom.expression.IoFunctionSet
 import wom.values.{WomEvaluatedCallInputs, WomValue}
 
@@ -26,14 +25,9 @@ object Command {
                   callEngineFunction: IoFunctionSet,
                   inputsPreProcessor: WomEvaluatedCallInputs => Try[WomEvaluatedCallInputs] =
                     (i: WomEvaluatedCallInputs) => Success(i),
-                  valueMapper: WomValue => WomValue,
-                  runtimeEnvironment: RuntimeEnvironment
+                  valueMapper: WomValue => WomValue
   ): ErrorOr[InstantiatedCommand] =
     inputsPreProcessor(jobDescriptor.evaluatedTaskInputs).toErrorOr flatMap { mappedInputs =>
-      jobDescriptor.taskCall.callable.instantiateCommand(mappedInputs,
-                                                         callEngineFunction,
-                                                         valueMapper,
-                                                         runtimeEnvironment
-      )
+      jobDescriptor.taskCall.callable.instantiateCommand(mappedInputs, callEngineFunction, valueMapper)
     }
 }

backend/src/main/scala/cromwell/backend/MinimumRuntimeSettings.scala

@@ -0,0 +1,13 @@
+package cromwell.backend
+
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric.Positive
+import eu.timepit.refined.refineMV
+import wdl4s.parser.MemoryUnit
+import wom.format.MemorySize
+
+case class MinimumRuntimeSettings(cores: Int Refined Positive = refineMV(1),
+                                  ram: MemorySize = MemorySize(4, MemoryUnit.GB),
+                                  outputPathSize: Long = Long.MaxValue,
+                                  tempPathSize: Long = Long.MaxValue
+)

backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala

@@ -37,7 +37,7 @@ import net.ceedubs.ficus.Ficus._
 import org.apache.commons.lang3.StringUtils
 import org.apache.commons.lang3.exception.ExceptionUtils
 import shapeless.Coproduct
-import wom.callable.{AdHocValue, CommandTaskDefinition, ContainerizedInputExpression, RuntimeEnvironment}
+import wom.callable.{AdHocValue, CommandTaskDefinition, ContainerizedInputExpression}
 import wom.expression.WomExpression
 import wom.graph.LocalName
 import wom.values._
@@ -141,7 +141,7 @@ trait StandardAsyncExecutionActor
 
   lazy val temporaryDirectory: String = configurationDescriptor.backendConfig.getOrElse(
     path = "temporary-directory",
-    default = s"""$$(mkdir -p "${runtimeEnvironment.tempPath}" && echo "${runtimeEnvironment.tempPath}")"""
+    default = """$(mktemp -d "$PWD"/tmp.XXXXXX)"""
   )
 
   val logJobIds: Boolean = true
@@ -431,7 +431,6 @@ trait StandardAsyncExecutionActor
       s"""export $k="$v""""
     } mkString ("", "\n", "\n")
 
-    val home = jobDescriptor.taskCall.callable.homeOverride.map(_(runtimeEnvironment)).getOrElse("$HOME")
     val shortId = jobDescriptor.workflowDescriptor.id.shortString
     // Give the out and error FIFO variables names that are unlikely to conflict with anything the user is doing.
     val (out, err) = (s"out$shortId", s"err$shortId")
@@ -479,7 +478,6 @@ trait StandardAsyncExecutionActor
           |$tmpDirPermissionsAdjustment
           |export _JAVA_OPTIONS=-Djava.io.tmpdir="$$tmpDir"
           |export TMPDIR="$$tmpDir"
-          |export HOME="$home"
           |
           |SCRIPT_PREAMBLE
           |
@@ -512,16 +510,6 @@ trait StandardAsyncExecutionActor
     )
   }
 
-  def runtimeEnvironmentPathMapper(env: RuntimeEnvironment): RuntimeEnvironment = {
-    def localize(path: String): String = (WomSingleFile(path) |> commandLineValueMapper).valueString
-    env.copy(outputPath = env.outputPath |> localize, tempPath = env.tempPath |> localize)
-  }
-
-  lazy val runtimeEnvironment: RuntimeEnvironment =
-    RuntimeEnvironmentBuilder(jobDescriptor.runtimeAttributes, jobPaths)(
-      standardParams.minimumRuntimeSettings
-    ) |> runtimeEnvironmentPathMapper
-
   /**
    * Turns WomFiles into relative paths.  These paths are relative to the working disk.
    *
@@ -685,8 +673,7 @@ trait StandardAsyncExecutionActor
       jobDescriptor,
       backendEngineFunctions,
       mutatingPreProcessor,
-      commandLineValueMapper,
-      runtimeEnvironment
+      commandLineValueMapper
     )
 
     def makeStringKeyedMap(list: List[(LocalName, WomValue)]): Map[String, WomValue] = list.toMap map { case (k, v) =>

docs/tutorials/HPCSlurmWithLocalScratch.md

@@ -3,6 +3,8 @@
 # Installing the Cromwell To Use Local Scratch Device
 #### These instructions are a community contribution
 
+### In the process of being updated as of 2024-02
+
 In this section we will install the Cromwell Workflow Management system and configure it, so it can use the local scratch device on the compute nodes.
 (these installations are done in a ```centos 8``` enviornment)
 

supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActor.scala

@@ -50,7 +50,7 @@ import org.apache.commons.csv.{CSVFormat, CSVPrinter}
 import org.apache.commons.io.output.ByteArrayOutputStream
 import wom.callable.Callable.OutputDefinition
 import wom.callable.MetaValueElement.{MetaValueElementBoolean, MetaValueElementObject}
-import wom.callable.{AdHocValue, RuntimeEnvironment}
+import wom.callable.AdHocValue
 import wom.core.FullyQualifiedName
 import wom.expression.{FileEvaluation, NoIoFunctionSet}
 import wom.values._
@@ -992,12 +992,6 @@ class GcpBatchAsyncBackendJobExecutionActor(override val standardParams: Standar
   override lazy val executeOrRecoverBackOff: SimpleExponentialBackoff =
     SimpleExponentialBackoff(initialInterval = 5.seconds, maxInterval = 20.seconds, multiplier = 1.1)
 
-  override lazy val runtimeEnvironment: RuntimeEnvironment =
-    RuntimeEnvironmentBuilder(jobDescriptor.runtimeAttributes,
-                              GcpBatchWorkingDisk.MountPoint,
-                              GcpBatchWorkingDisk.MountPoint
-    )(standardParams.minimumRuntimeSettings)
-
   protected def sendIncrementMetricsForReferenceFiles(referenceInputFilesOpt: Option[Set[GcpBatchInput]]): Unit =
     referenceInputFilesOpt match {
       case Some(referenceInputFiles) =>

supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala

@@ -59,7 +59,7 @@ import mouse.all._
 import shapeless.Coproduct
 import wom.callable.Callable.OutputDefinition
 import wom.callable.MetaValueElement.{MetaValueElementBoolean, MetaValueElementObject}
-import wom.callable.{AdHocValue, RuntimeEnvironment}
+import wom.callable.AdHocValue
 import wom.core.FullyQualifiedName
 import wom.expression.{FileEvaluation, NoIoFunctionSet}
 import wom.types.{WomArrayType, WomSingleFileType}
@@ -146,12 +146,6 @@ class PipelinesApiAsyncBackendJobExecutionActor(override val standardParams: Sta
   override lazy val executeOrRecoverBackOff: SimpleExponentialBackoff =
     SimpleExponentialBackoff(initialInterval = 3 seconds, maxInterval = 20 seconds, multiplier = 1.1)
 
-  override lazy val runtimeEnvironment: RuntimeEnvironment =
-    RuntimeEnvironmentBuilder(jobDescriptor.runtimeAttributes,
-                              PipelinesApiWorkingDisk.MountPoint,
-                              PipelinesApiWorkingDisk.MountPoint
-    )(standardParams.minimumRuntimeSettings)
-
   protected lazy val cmdInput: PipelinesApiFileInput =
     PipelinesApiFileInput(PipelinesApiJobPaths.JesExecParamName,
                           pipelinesApiCallPaths.script,

supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala

@@ -4,7 +4,6 @@ import java.nio.file.FileAlreadyExistsException
 import java.time.Instant
 
 import common.validation.Validation._
-import cromwell.backend.RuntimeEnvironmentBuilder
 import cromwell.backend.impl.sfs.config.ConfigConstants._
 import cromwell.backend.sfs._
 import cromwell.backend.standard.{StandardAsyncExecutionActorParams, StandardAsyncJob}
@@ -104,10 +103,8 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut
       if !inputs.contains(optional.localName.value)
     } yield optional -> WomOptionalValue.none(optional.womType.memberType)
 
-    val runtimeEnvironment =
-      RuntimeEnvironmentBuilder(jobDescriptor.runtimeAttributes, jobPaths)(standardParams.minimumRuntimeSettings)
     val allInputs = providedWomInputs ++ optionalsForciblyInitializedToNone
-    val womInstantiation = taskDefinition.instantiateCommand(allInputs, NoIoFunctionSet, identity, runtimeEnvironment)
+    val womInstantiation = taskDefinition.instantiateCommand(allInputs, NoIoFunctionSet, identity)
 
     val command = womInstantiation.toTry.get.commandString
     jobLogger.info(s"executing: $command")

wdl/model/draft2/src/main/scala/wdl/draft2/model/command/WdlCommandPart.scala

@@ -4,7 +4,6 @@ import common.validation.ErrorOr.ErrorOr
 import wdl.draft2.model.Declaration
 import wdl.draft2.model.expression.{WdlFunctions, WdlStandardLibraryFunctions}
 import wdl.shared.FileSizeLimitationConfig
-import wom.callable.RuntimeEnvironment
 import wom.expression.IoFunctionSet
 import wom.graph.LocalName
 import wom.values.WomValue
@@ -20,8 +19,7 @@ trait WdlCommandPart extends CommandPart {
 
   override def instantiate(inputsMap: Map[LocalName, WomValue],
                            functions: IoFunctionSet,
-                           valueMapper: WomValue => WomValue,
-                           runtimeEnvironment: RuntimeEnvironment
+                           valueMapper: WomValue => WomValue
   ): ErrorOr[List[InstantiatedCommand]] = {
     val wdlFunctions =
       WdlStandardLibraryFunctions.fromIoFunctionSet(functions, FileSizeLimitationConfig.fileSizeLimitationConfig)

wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wom/CommandPartElementToWomCommandPart.scala

@@ -9,7 +9,6 @@ import wdl.model.draft3.graph.{ExpressionValueConsumer, GeneratedValueHandle}
 import wdl.model.draft3.graph.expression._
 import wdl.transforms.base.linking.graph.LinkedGraphMaker
 import wdl.transforms.base.wdlom2wom.expression.WdlomWomExpression
-import wom.callable.RuntimeEnvironment
 import wom.expression.{IoFunctionSet, WomExpression}
 import wom.graph.LocalName
 import wom.types.{WomArrayType, WomPrimitiveType, WomType}
@@ -52,8 +51,7 @@ case class WdlomWomStringCommandPart(stringCommandPartElement: StringCommandPart
   override def toString: String = stringCommandPartElement.value
   override def instantiate(inputsMap: Map[LocalName, WomValue],
                            functions: IoFunctionSet,
-                           valueMapper: WomValue => WomValue,
-                           runtimeEnvironment: RuntimeEnvironment
+                           valueMapper: WomValue => WomValue
   ): ErrorOr[List[InstantiatedCommand]] = List(InstantiatedCommand(stringCommandPartElement.value)).validNel
 }
 
@@ -65,8 +63,7 @@ case class WdlomWomPlaceholderCommandPart(attributes: PlaceholderAttributeSet, e
 
   override def instantiate(inputsMap: Map[LocalName, WomValue],
                            functions: IoFunctionSet,
-                           valueMapper: WomValue => WomValue,
-                           runtimeEnvironment: RuntimeEnvironment
+                           valueMapper: WomValue => WomValue
   ): ErrorOr[List[InstantiatedCommand]] = {
     val inputsValues = inputsMap map { case (localName, value) => localName.value -> value }
     expression.evaluateValueForPlaceholder(inputsValues,

wom/src/main/scala/wom/CommandPart.scala

@@ -1,15 +1,13 @@
 package wom
 
 import common.validation.ErrorOr.ErrorOr
-import wom.callable.RuntimeEnvironment
 import wom.expression.IoFunctionSet
 import wom.graph.LocalName
 import wom.values.WomValue
 
 trait CommandPart {
   def instantiate(inputsMap: Map[LocalName, WomValue],
                   functions: IoFunctionSet,
-                  valueMapper: WomValue => WomValue,
-                  runtimeEnvironment: RuntimeEnvironment
+                  valueMapper: WomValue => WomValue
   ): ErrorOr[List[InstantiatedCommand]]
 }

wom/src/main/scala/wom/callable/CommandTaskDefinition.scala

@@ -92,7 +92,6 @@ sealed trait CommandTaskDefinition extends TaskDefinition {
   def adHocFileCreation: Set[ContainerizedInputExpression]
   def environmentExpressions: Map[String, WomExpression]
   def additionalGlob: Option[WomGlobFile]
-  def homeOverride: Option[RuntimeEnvironment => String]
 
   /**
     * Provides a custom way to evaluate outputs of the task definition.
@@ -104,8 +103,7 @@ sealed trait CommandTaskDefinition extends TaskDefinition {
 
   def instantiateCommand(taskInputs: WomEvaluatedCallInputs,
                          functions: IoFunctionSet,
-                         valueMapper: WomValue => WomValue,
-                         runtimeEnvironment: RuntimeEnvironment
+                         valueMapper: WomValue => WomValue
   ): ErrorOr[InstantiatedCommand] = {
 
     val inputsByLocalName = taskInputs map { case (k, v) => k.localName -> v }
@@ -115,7 +113,7 @@ sealed trait CommandTaskDefinition extends TaskDefinition {
     // Just raw command parts, no separators.
     val rawCommandParts: List[ErrorOr[InstantiatedCommand]] =
       commandTemplate(taskInputs).toList.flatMap { commandPart =>
-        commandPart.instantiate(inputsByLocalName, functions, valueMapper, runtimeEnvironment).sequence
+        commandPart.instantiate(inputsByLocalName, functions, valueMapper).sequence
       }
 
     // Add separator command parts and monoid smash down to one `ErrorOr[InstantiatedCommand]`.
@@ -163,7 +161,6 @@ final case class CallableTaskDefinition(name: String,
                                         additionalGlob: Option[WomGlobFile] = None,
                                         private[wom] val customizedOutputEvaluation: OutputEvaluationFunction =
                                           OutputEvaluationFunction.none,
-                                        homeOverride: Option[RuntimeEnvironment => String] = None,
                                         dockerOutputDirectory: Option[String] = None,
                                         override val sourceLocation: Option[SourceFileLocation]
 ) extends CommandTaskDefinition {
@@ -198,7 +195,6 @@ final case class ExecutableTaskDefinition private (callableTaskDefinition: Calla
   override def additionalGlob = callableTaskDefinition.additionalGlob
   override private[wom] def customizedOutputEvaluation = callableTaskDefinition.customizedOutputEvaluation
   override def toExecutable = this.validNel
-  override def homeOverride = callableTaskDefinition.homeOverride
   override def dockerOutputDirectory = callableTaskDefinition.dockerOutputDirectory
 }