Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

womtool support parametermeta command #7556

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package wdl.transforms.base.wdlom2wom

import cats.data.NonEmptyList
import cats.data.Validated.{Invalid, Valid}
import cats.instances.list._
import cats.syntax.apply._
import cats.syntax.traverse._
Expand Down Expand Up @@ -112,28 +110,6 @@ object TaskDefinitionElementToWomTaskDefinition extends Util {
)
}
}
private def validateParameterMetaEntries(parameterMetaSectionElement: Option[ParameterMetaSectionElement],
inputs: Option[InputsSectionElement],
outputs: Option[OutputsSectionElement]
): ErrorOr[Unit] = {
val validKeys: List[String] =
inputs.toList.flatMap(_.inputDeclarations.map(_.name)) ++ outputs.toList.flatMap(_.outputs.map(_.name))
val errors = parameterMetaSectionElement.toList.flatMap { pmse =>
val keys = pmse.metaAttributes.keySet.toList
val duplicationErrors = keys.groupBy(identity).collect {
case (name, list) if list.size > 1 => s"Found ${list.size} parameter meta entries for '$name' (expected 0 or 1)"
}
val notValidKeyErrors = keys.collect {
case name if !validKeys.contains(name) =>
s"Invalid parameter_meta entry for '$name': not an input or output parameter"
}
duplicationErrors.toList ++ notValidKeyErrors
}
NonEmptyList.fromList(errors) match {
case Some(nel) => Invalid(nel)
case None => Valid(())
}
}

def eliminateInputDependencies(
a: TaskDefinitionElementToWomInputs
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package wdl.transforms.base.wdlom2wom

import wdl.model.draft3.elements.{MetaSectionElement, ParameterMetaSectionElement}
import cats.data.NonEmptyList
import cats.data.Validated.{Invalid, Valid}
import common.validation.ErrorOr.ErrorOr
import wdl.model.draft3.elements.{InputsSectionElement, MetaSectionElement, OutputsSectionElement, ParameterMetaSectionElement}

trait Util {

Expand All @@ -11,4 +14,27 @@ trait Util {
(metaMap, parameterMetaMap)
}

def validateParameterMetaEntries(parameterMetaSectionElement: Option[ParameterMetaSectionElement],
inputs: Option[InputsSectionElement],
outputs: Option[OutputsSectionElement]
): ErrorOr[Unit] = {
val validKeys: List[String] =
inputs.toList.flatMap(_.inputDeclarations.map(_.name)) ++ outputs.toList.flatMap(_.outputs.map(_.name))
val errors = parameterMetaSectionElement.toList.flatMap { pmse =>
val keys = pmse.metaAttributes.keySet.toList
val duplicationErrors = keys.groupBy(identity).collect {
case (name, list) if list.size > 1 => s"Found ${list.size} parameter meta entries for '$name' (expected 0 or 1)"
}
val notValidKeyErrors = keys.collect {
case name if !validKeys.contains(name) =>
s"Invalid parameter_meta entry for '$name': not an input or output parameter"
}
duplicationErrors.toList ++ notValidKeyErrors
}
NonEmptyList.fromList(errors) match {
case Some(nel) => Invalid(nel)
case None => Valid(())
}
}

}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package wdl.transforms.base.wdlom2wom

import cats.syntax.validated._
import common.validation.ErrorOr
import common.validation.ErrorOr.{ErrorOr, _}
import wdl.model.draft3.elements.ExpressionElement.{ArrayLiteral, IdentifierLookup, SelectFirst}
import wdl.model.draft3.elements._
Expand All @@ -16,7 +17,7 @@ import wom.callable.MetaValueElement.MetaValueElementBoolean
import wom.callable.{Callable, WorkflowDefinition}
import wom.graph.GraphNodePort.OutputPort
import wom.graph.expression.AnonymousExpressionNode
import wom.graph.{CallNode, Graph => WomGraph, GraphNode, WomIdentifier}
import wom.graph.{CallNode, GraphNode, WomIdentifier, Graph => WomGraph}
import wom.types.WomType

object WorkflowDefinitionElementToWomWorkflowDefinition extends Util {
Expand Down Expand Up @@ -77,9 +78,16 @@ object WorkflowDefinitionElementToWomWorkflowDefinition extends Util {
innerGraph
}

(withDefaultOutputs map { ig =>
WorkflowDefinition(a.definitionElement.name, ig, meta, parameterMeta, b.definitionElement.sourceLocation)
}).contextualizeErrors(s"process workflow definition '${a.definitionElement.name}'")
val conversion = (
withDefaultOutputs,
validateParameterMetaEntries(a.definitionElement.parameterMetaSection, a.definitionElement.inputsSection, a.definitionElement.outputsSection)
) flatMapN {
(ig, _) => ErrorOr[WorkflowDefinition] {
WorkflowDefinition(a.definitionElement.name, ig, meta, parameterMeta, b.definitionElement.sourceLocation)
}
}

conversion.contextualizeErrors(s"process workflow definition '${a.definitionElement.name}'")
}

final case class GraphLikeConvertInputs(graphElements: Set[WorkflowGraphElement],
Expand Down
2 changes: 2 additions & 0 deletions womtool/src/main/scala/womtool/WomtoolMain.scala
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import womtool.graph.WomGraph
import womtool.input.WomGraphMaker
import womtool.inputs.Inputs
import womtool.outputs.Outputs
import womtool.parametermeta.ParameterMeta
import womtool.validate.Validate

import scala.util.{Failure, Success}
Expand Down Expand Up @@ -53,6 +54,7 @@ object WomtoolMain extends App with StrictLogging {
case o: OutputsCommandLine => Outputs.outputsJson(o.workflowSource)
case g: WomtoolGraphCommandLine => graph(g.workflowSource)
case g: WomtoolWomGraphCommandLine => womGraph(g.workflowSource)
case pm: ParameterMetaCommandLine => ParameterMeta.parameterMetaInfoJson(pm.workflowSource)
case _ => BadUsageTermination(WomtoolCommandLineParser.instance.usage)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ final case class InputsCommandLine(workflowSource: Path, showOptionals: Boolean)
final case class OutputsCommandLine(workflowSource: Path) extends ValidatedWomtoolCommandLine
final case class WomtoolGraphCommandLine(workflowSource: Path) extends ValidatedWomtoolCommandLine
final case class WomtoolWomGraphCommandLine(workflowSource: Path) extends ValidatedWomtoolCommandLine
final case class ParameterMetaCommandLine(workflowSource: Path) extends ValidatedWomtoolCommandLine

sealed trait WomtoolCommand

Expand All @@ -31,6 +32,7 @@ object WomtoolCommand {
case object Outputs extends WomtoolCommand
case object Graph extends WomtoolCommand
case object WomGraph extends WomtoolCommand
case object ParameterMeta extends WomtoolCommand
}

sealed trait HighlightMode
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ object WomtoolCommandLineParser {
Option(WomtoolGraphCommandLine(mainFile))
case PartialWomtoolCommandLineArguments(Some(WomGraph), Some(mainFile), None, None, None, None) =>
Option(WomtoolWomGraphCommandLine(mainFile))
case PartialWomtoolCommandLineArguments(Some(ParameterMeta), Some(mainFile), None, None, None, None) =>
Option(ParameterMetaCommandLine(mainFile))
case _ => None
}
}
Expand Down Expand Up @@ -111,4 +113,8 @@ class WomtoolCommandLineParser extends scopt.OptionParser[PartialWomtoolCommandL
"(Advanced) Generate and output a graph visualization of Cromwell's internal Workflow Object Model structure for this workflow in .dot format" + System.lineSeparator
)

cmd("parametermeta")
.action((_, c) => c.copy(command = Option(ParameterMeta)))
.text("Generate and output parameter metadata in JSON for this workflow" + System.lineSeparator)

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package womtool.parametermeta

import common.Checked
import cromwell.core.path.Path
import spray.json.{JsNull, JsObject, JsonWriter, enrichAny}
import spray.json.DefaultJsonProtocol._
import wom.callable.{MetaValueElement, TaskDefinition, WorkflowDefinition}
import wom.executable.WomBundle
import womtool.WomtoolMain.{SuccessfulTermination, Termination, UnsuccessfulTermination}
import womtool.input.WomGraphMaker

object ParameterMeta {
def parameterMetaInfoJson(main: Path): Termination = {
WomGraphMaker.getBundle(main) match {
case Right(b) =>
getParameterMeta(b) match {
case Right(inputs) =>
SuccessfulTermination(inputs.toJson(parameterMetaInfoJsonWriter()).prettyPrint)
case Left(errors) =>
UnsuccessfulTermination(errors.toList.mkString(System.lineSeparator))
}
case Left(errors) =>
UnsuccessfulTermination(errors.toList.mkString(System.lineSeparator))
}
}

private final case class ParameterMetaInfo(inputs: Map[String, Option[MetaValueElement]],
outputs: Map[String, Option[MetaValueElement]])

private def parameterMetaInfoJsonWriter(): JsonWriter[ParameterMetaInfo] = info => {
val inputs = info.inputs.toJson(parameterMetaJsonWriter())
val outputs = info.outputs.toJson(parameterMetaJsonWriter())
JsObject(
"inputs" -> inputs,
"outputs" -> outputs
)
}

private def parameterMetaJsonWriter(): JsonWriter[Map[String, Option[MetaValueElement]]] = m => {
m.collect {
case (name, Some(meta)) => name -> meta.toJson(metaValueElementJsonWriter())
}.toJson
}

private def metaValueElementJsonWriter(): JsonWriter[MetaValueElement] = {
case MetaValueElement.MetaValueElementNull => JsNull
case MetaValueElement.MetaValueElementBoolean(value) => value.toJson
case MetaValueElement.MetaValueElementFloat(value) => value.toJson
case MetaValueElement.MetaValueElementInteger(value) => value.toJson
case MetaValueElement.MetaValueElementString(value) => value.toJson
case MetaValueElement.MetaValueElementObject(value) =>
value.view.mapValues(m => m.toJson(metaValueElementJsonWriter())).toMap.toJson
case MetaValueElement.MetaValueElementArray(value) =>
value.map(m => m.toJson(metaValueElementJsonWriter())).toJson
}

private def getParameterMeta(b: WomBundle): Checked[ParameterMetaInfo] = {
b.toExecutableCallable.map(primaryCallable => {
// inputs the same as womtool inputs
val inputs = primaryCallable.graph.externalInputNodes.map(inputNode => {
val parameterFullName = inputNode.nameInInputSet
parameterFullName -> findParameterMetaValue(b, parameterFullName)
}).toMap
// outputs the same as womtool outputs
val outputs = primaryCallable.graph.outputNodes.map(outputNode => {
val parameterFullName = outputNode.fullyQualifiedName
parameterFullName -> findParameterMetaValue(b, parameterFullName)
}).toMap
ParameterMetaInfo(inputs, outputs)
})
}

private def findParameterMetaValue(b: WomBundle, parameterFullName: String): Option[MetaValueElement] = {
val parts = parameterFullName.split("\\.")
val parameterName = parts(parts.length - 1)
val callableName: Option[String] = if (parts.length >= 2) Some(parts(parts.length - 2)) else None
callableName.flatMap(name => {
b.allCallables.get(name) match {
case Some(w: WorkflowDefinition) => w.parameterMeta.get(parameterName)
case Some(t: TaskDefinition) => t.parameterMeta.get(parameterName)
case _ => None
}
})
}

}
Loading