diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fa01827 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +tags +.idea/ +target +.artifactory +*~ diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..6acd685 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,27 @@ +Copyright (c) 2015, Broad Institute, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name Broad Institute, Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE \ No newline at end of file diff --git a/README.md b/README.md index 3989b31..8803dd5 100644 --- a/README.md +++ b/README.md @@ -1 +1,170 @@ -# wdltool \ No newline at end of file +Command line utilities for interacting with WDL + + + +* [Requirements](#requirements) +* [Building](#building) +* [Command Line Usage](#command-line-usage) + * [validate](#validate) + * [inputs](#inputs) + * [highlight](#highlight) + * [parse](#parse) +* [Getting Started with WDL](#getting-started-with-wdl) + + + +# Requirements + +The following is the toolchain used for development of wdltool. Other versions may work, but these are recommended. + +* [Scala 2.11.7](http://www.scala-lang.org/news/2.11.7) +* [SBT 0.13.8](https://github.com/sbt/sbt/releases/tag/v0.13.8) +* [Java 8](http://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html) + +# Building + +`sbt assembly` will build a runnable JAR in `target/scala-2.11/` + +Tests are run via `sbt test`. Note that the tests do require Docker to be running. To test this out while downloading the Ubuntu image that is required for tests, run `docker pull ubuntu:latest` prior to running `sbt test` + +# Command Line Usage + +Run the JAR file with no arguments to get the usage message: + +``` +$ java -jar wdltool.jar +java -jar wdltool.jar + +Actions: +validate + + Performs full validation of the WDL file including syntax + and semantic checking + +inputs + + Write a JSON skeleton file of the inputs needed for this + workflow. Fill in the values in this JSON document and + pass it in to the 'run' subcommand. + +highlight + + Reformats and colorizes/tags a WDL file. The second + parameter is the output type. "html" will output the WDL + file with tags around elements. "console" mode + will output colorized text to the terminal + +parse + + Compares a WDL file against the grammar and writes out an + abstract syntax tree if it is valid, and a syntax error + otherwise. Note that higher-level AST checks are not done + via this sub-command and the 'validate' subcommand should + be used for full validation +``` + +## validate + +Given a WDL file, this runs the full syntax checker over the file and resolves imports in the process. If any syntax errors are found, they are written out. Otherwise the program exits. + +Error if a `call` references a task that doesn't exist: + +``` +$ java -jar wdltool.jar validate 2.wdl +ERROR: Call references a task (BADps) that doesn't exist (line 22, col 8) + + call BADps + ^ + +``` + +Error if namespace and task have the same name: + +``` +$ java -jar wdltool.jar validate 5.wdl +ERROR: Task and namespace have the same name: + +Task defined here (line 3, col 6): + +task ps { + ^ + +Import statement defined here (line 1, col 20): + +import "ps.wdl" as ps + ^ +``` + +## inputs + +Examine a WDL file with one workflow in it, compute all the inputs needed for that workflow and output a JSON template that the user can fill in with values. The keys in this document should remain unchanged. The values tell you what type the parameter is expecting. For example, if the value were `Array[String]`, then it's expecting a JSON array of JSON strings, like this: `["string1", "string2", "string3"]` + +``` +$ java -jar wdltool.jar inputs 3step.wdl +{ + "three_step.cgrep.pattern": "String" +} +``` + +This inputs document is used as input to the `run` subcommand. + +## highlight + +Formats a WDL file and semantically tags it. This takes a second parameter (`html` or `console`) which determines what the output format will be. + +test.wdl +``` +task abc { + String in + command { + echo ${in} + } + output { + String out = read_string(stdout()) + } +} + +workflow wf { + call abc +} +``` + +## parse + +Given a WDL file input, this does grammar level syntax checks and writes out the resulting abstract syntax tree. + +``` +$ echo "workflow wf {}" | java -jar wdltool.jar parse /dev/stdin +(Document: + imports=[], + definitions=[ + (Workflow: + name=task abc { + String in + command { + echo ${in} + } + output { + String out = read_string(stdout()) + } +} + +workflow wf { + call abc +} +``` + +# Getting Started with WDL + +For many examples on how to use WDL see [the WDL site](https://github.com/broadinstitute/wdl/tree/develop#getting-started-with-wdl) diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..298bbbd --- /dev/null +++ b/build.sbt @@ -0,0 +1,70 @@ +import com.typesafe.sbt.GitPlugin.autoImport._ +import sbt.Keys._ +import sbtassembly.MergeStrategy +import com.typesafe.sbt.SbtGit.GitCommand + +name := "wdltool" + +organization := "org.broadinstitute" + +scalaVersion := "2.11.7" + +// Upcoming release, or current if we're on the master branch +git.baseVersion := "0.1" + +// Shorten the git commit hash +git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) } + +// Travis will deploy tagged releases, add -SNAPSHOT for all local builds +git.gitUncommittedChanges := true + +versionWithGit + +assemblyJarName in assembly := "wdltool" + git.baseVersion.value + ".jar" + +logLevel in assembly := Level.Info + + +libraryDependencies ++= Seq( + "org.broadinstitute" %% "wdl4s" % "0.1", + //---------- Test libraries -------------------// + "org.scalatest" %% "scalatest" % "2.2.5" % Test +) + +val customMergeStrategy: String => MergeStrategy = { + case x if Assembly.isConfigFile(x) => + MergeStrategy.concat + case PathList(ps@_*) if Assembly.isReadme(ps.last) || Assembly.isLicenseFile(ps.last) => + MergeStrategy.rename + case PathList("META-INF", path@_*) => + path map { + _.toLowerCase + } match { + case ("manifest.mf" :: Nil) | ("index.list" :: Nil) | ("dependencies" :: Nil) => + MergeStrategy.discard + case ps@(x :: xs) if ps.last.endsWith(".sf") || ps.last.endsWith(".dsa") => + MergeStrategy.discard + case "plexus" :: xs => + MergeStrategy.discard + case "spring.tooling" :: xs => + MergeStrategy.discard + case "services" :: xs => + MergeStrategy.filterDistinctLines + case ("spring.schemas" :: Nil) | ("spring.handlers" :: Nil) => + MergeStrategy.filterDistinctLines + case _ => MergeStrategy.deduplicate + } + case "asm-license.txt" | "overview.html" | "cobertura.properties" => + MergeStrategy.discard + case _ => MergeStrategy.deduplicate +} + +assemblyMergeStrategy in assembly := customMergeStrategy + +// The reason why -Xmax-classfile-name is set is because this will fail +// to build on Docker otherwise. The reason why it's 200 is because it +// fails if the value is too close to 256 (even 254 fails). For more info: +// +// https://github.com/sbt/sbt-assembly/issues/69 +// https://github.com/scala/pickling/issues/10 +scalacOptions ++= Seq("-deprecation", "-unchecked", "-feature", "-Xmax-classfile-name", "200") diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..20755cc --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,9 @@ +addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.8.5") + +addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.0") + +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") + +addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0") + +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.1") diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala new file mode 100644 index 0000000..59c9943 --- /dev/null +++ b/src/main/scala/wdltool/Main.scala @@ -0,0 +1,134 @@ +package wdltool + +import java.io.{File => JFile} + +import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} +import wdl4s.{WdlNamespace, NamespaceWithWorkflow, AstTools} +import spray.json._ + +import scala.util.{Failure, Success, Try} + +object Main extends App { + sealed trait Termination { + def output: String + def returnCode: Int + } + + case class SuccessfulTermination(output: String) extends Termination { + override val returnCode = 0 + } + + case class UnsuccessfulTermination(output: String) extends Termination { + override val returnCode = 1 + } + + case object BadUsageTermination extends Termination { + override val returnCode = 1 + override val output = UsageMessage + } + + def dispatchCommand(args: Seq[String]): Termination = { + getAction(args) match { + case Some(x) if x == Actions.Validate => validate(args.tail) + case Some(x) if x == Actions.Highlight => highlight(args.tail) + case Some(x) if x == Actions.Inputs => inputs(args.tail) + case Some(x) if x == Actions.Parse => parse(args.tail) + case _ => BadUsageTermination + } + } + + def validate(args: Seq[String]): Termination = { + continueIf(args.length == 1) { + loadWdl(args.head) { _ => SuccessfulTermination("") } + } + } + + def highlight(args: Seq[String]): Termination = { + continueIf(args.length == 2 && Seq("html", "console").contains(args(1))) { + loadWdl(args.head) { namespace => + val formatter = new SyntaxFormatter(if (args(1) == "html") HtmlSyntaxHighlighter else AnsiSyntaxHighlighter) + SuccessfulTermination(formatter.format(namespace)) + } + } + } + + def inputs(args: Seq[String]): Termination = { + continueIf(args.length == 1) { + loadWdl(args.head) { namespace => + import wdl4s.types.WdlTypeJsonFormatter._ + val msg = namespace match { + case x: NamespaceWithWorkflow => x.workflow.inputs.toJson.prettyPrint + case _ => "WDL does not have a local workflow" + } + + SuccessfulTermination(msg) + } + } + } + + def parse(args: Seq[String]): Termination = { + continueIf(args.length == 1) { + SuccessfulTermination(AstTools.getAst(new JFile(args.head)).toPrettyString) + } + } + + private[this] def continueIf(valid: => Boolean)(block: => Termination): Termination = if (valid) block else BadUsageTermination + + private[this] def loadWdl(path: String)(f: WdlNamespace => Termination): Termination = { + Try(WdlNamespace.load(new JFile(path))) match { + case Success(namespace) => f(namespace) + case Failure(t) => UnsuccessfulTermination(t.getMessage) + } + } + + private def getAction(args: Seq[String]): Option[Actions.Value] = for { + arg <- args.headOption + argCapitalized = arg.capitalize + action <- Actions.values find (_.toString == argCapitalized) + } yield action + + object Actions extends Enumeration { + val Parse, Validate, Highlight, Inputs = Value + } + + val UsageMessage = """ + |java -jar wdltool.jar + | + |Actions: + |validate + | + | Performs full validation of the WDL file including syntax + | and semantic checking + | + |inputs + | + | Print a JSON skeleton file of the inputs needed for this + | workflow. Fill in the values in this JSON document and + | pass it in to the 'run' subcommand. + | + |highlight + | + | Reformats and colorizes/tags a WDL file. The second + | parameter is the output type. "html" will output the WDL + | file with tags around elements. "console" mode + | will output colorized text to the terminal + | + |parse + | + | Compares a WDL file against the grammar and prints out an + | abstract syntax tree if it is valid, and a syntax error + | otherwise. Note that higher-level AST checks are not done + | via this sub-command and the 'validate' subcommand should + | be used for full validation + """.stripMargin + + val termination = dispatchCommand(args) + + termination match { + case SuccessfulTermination(s) => println(s) + case UnsuccessfulTermination(s) => Console.err.println(s) + case BadUsageTermination => Console.err.println(UsageMessage) + } + + termination.returnCode +} diff --git a/src/test/scala/wdltool/MainSpec.scala b/src/test/scala/wdltool/MainSpec.scala new file mode 100644 index 0000000..01c69c9 --- /dev/null +++ b/src/test/scala/wdltool/MainSpec.scala @@ -0,0 +1,184 @@ +package wdltool + +import java.nio.file.{Paths, Path} +import better.files._ +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import wdltool.SampleWdl.{EmptyWorkflow, EmptyTask, EmptyInvalid, ThreeStep} +import MainSpec._ + +class MainSpec extends FlatSpec with Matchers with BeforeAndAfterAll { + + import Main._ + + behavior of "Main" + + val threeStep = ThreeStep.wdlSource() + + it should "print usage" in { + Main.dispatchCommand(Seq.empty[String]) shouldBe BadUsageTermination + } + + it should "validate" in { + testWdl(ThreeStep) { wdlAndInputs => + Main.dispatchCommand(Seq("validate", wdlAndInputs.wdl)) shouldBe SuccessfulTermination("") + } + } + + it should "not validate invalid wdl" in { + testWdl(EmptyInvalid) { wdlAndInputs => + val res = Main.dispatchCommand(Seq("validate", wdlAndInputs.wdl)) + assert(res.isInstanceOf[UnsuccessfulTermination]) + res.output should include("Finished parsing without consuming all tokens") + } + } + + it should "parse" in { + testWdl(ThreeStep) { wdlAndInputs => + val res = Main.dispatchCommand(Seq("parse", wdlAndInputs.wdl)) + assert(res.isInstanceOf[SuccessfulTermination]) + res.output should include("(Document:") + } + } + + it should "highlight" in { + testWdl(ThreeStep) { wdlAndInputs => + val res = Main.dispatchCommand(Seq("highlight", wdlAndInputs.wdl, "html")) + assert(res.isInstanceOf[SuccessfulTermination]) + res.output.stripLineEnd should be(HighlightedWdlHtml) + } + } + + it should "highlight using console highlighting" in { + testWdl(EmptyWorkflow) { wdlAndInputs => + val res = Main.dispatchCommand(Seq("highlight", wdlAndInputs.wdl, "console")) + assert(res.isInstanceOf[SuccessfulTermination]) + res.output.stripLineEnd should include("empty_workflow") + } + } + + it should "return inputs" in { + testWdl(ThreeStep) { wdlAndInputs => + val res = Main.dispatchCommand(Seq("inputs", wdlAndInputs.wdl)) + assert(res.isInstanceOf[SuccessfulTermination]) + res.output should include("\"three_step.cgrep.pattern\"") + } + } + + it should "not return inputs when there is no workflow" in { + testWdl(EmptyTask) { wdlAndInputs => + val res = Main.dispatchCommand(Seq("inputs", wdlAndInputs.wdl)) + assert(res.isInstanceOf[SuccessfulTermination]) + res.output should include("WDL does not have a local workflow") + } + } +} + +object MainSpec { + /** + * Tests running a sample wdl, providing the inputs, and cleaning up the temp files only if no exceptions occur. + * + * @param sampleWdl The sample wdl to run. + * @param optionsJson Optional json for the options file. + * @param block The block provided the inputs, returning some value. + * @tparam T The return type of the block. + * @return The result of running the block. + */ + def testWdl[T](sampleWdl: SampleWdl, optionsJson: String = "{}")(block: WdlAndInputs => T): T = { + val wdlAndInputs = WdlAndInputs(sampleWdl, optionsJson) + val result = block(wdlAndInputs) + wdlAndInputs.deleteTempFiles() + result + } + + val HighlightedWdlHtml = + """task ps { + | command { + | ps + | } + | output { + | File procs = stdout() + | } + |} + | + |task cgrep { + | String pattern + | File in_file + | command { + | grep '${pattern}' ${in_file} | wc -l + | } + | output { + | Int count = read_int(stdout()) + | } + |} + | + |task wc { + | File in_file + | command { + | cat ${in_file} | wc -l + | } + | output { + | Int count = read_int(stdout()) + | } + |} + | + |workflow three_step { + | call ps + | call cgrep { + | input: in_file=ps.procs + | } + | call wc { + | input: in_file=ps.procs + | } + |}""".stripMargin + + /** + * Create a temporary wdl file and inputs for the sampleWdl. + * When the various properties are lazily accessed, they are also registered for deletion after the suite completes. + */ + case class WdlAndInputs(sampleWdl: SampleWdl, optionsJson: String = "{}") { + // Track all the temporary files we create, and delete them after the test. + private var tempFiles = Vector.empty[Path] + + lazy val wdlPath: Path = { + val path = File.newTemp(s"${sampleWdl.name}.", ".wdl").path + tempFiles :+= path + path write sampleWdl.wdlSource("") + path + } + + lazy val wdl = wdlPath.fullPath + + lazy val inputsPath = { + val path = swapExt(wdlPath, ".wdl", ".inputs") + tempFiles :+= path + path write sampleWdl.wdlJson + path + } + + lazy val inputs = inputsPath.fullPath + + lazy val optionsPath = { + val path = swapExt(wdlPath, ".wdl", ".options") + tempFiles :+= path + path write optionsJson + path + } + + lazy val options = optionsPath.fullPath + + lazy val metadataPath = { + val path = swapExt(wdlPath, ".wdl", ".metadata.json") + tempFiles :+= path + path.toAbsolutePath + } + + lazy val metadata = metadataPath.fullPath + + def deleteTempFiles() = tempFiles.foreach(_.delete(ignoreIOExceptions = true)) + } + + def swapExt(filePath: Path, oldExt: String, newExt: String): Path = { + Paths.get(filePath.toString.stripSuffix(oldExt) + newExt) + } +} + diff --git a/src/test/scala/wdltool/SampleWdl.scala b/src/test/scala/wdltool/SampleWdl.scala new file mode 100644 index 0000000..d2179b9 --- /dev/null +++ b/src/test/scala/wdltool/SampleWdl.scala @@ -0,0 +1,142 @@ +package wdltool + +import java.io.{FileWriter, File} +import java.nio.file.{Files, Path} +import spray.json._ + +import wdl4s._ +import wdl4s.values._ + +import scala.language.postfixOps + +trait SampleWdl { + def wdlSource(runtime: String = ""): WdlSource + def rawInputs: WorkflowRawInputs + + def name = getClass.getSimpleName.stripSuffix("$") + + implicit object AnyJsonFormat extends JsonFormat[Any] { + def write(x: Any) = x match { + case n: Int => JsNumber(n) + case s: String => JsString(s) + case b: Boolean => if(b) JsTrue else JsFalse + case s: Seq[Any] => JsArray(s map {_.toJson} toVector) + case a: WdlArray => write(a.value) + case s: WdlString => JsString(s.value) + case i: WdlInteger => JsNumber(i.value) + case f: WdlFloat => JsNumber(f.value) + case f: WdlFile => JsString(f.value) + } + def read(value: JsValue) = throw new NotImplementedError(s"Reading JSON not implemented: $value") + } + + implicit object RawInputsJsonFormat extends JsonFormat[WorkflowRawInputs] { + def write(inputs: WorkflowRawInputs) = JsObject(inputs map { case (k, v) => k -> v.toJson }) + def read(value: JsValue) = throw new NotImplementedError(s"Reading JSON not implemented: $value") + } + + def wdlJson: WdlJson = rawInputs.toJson.prettyPrint + + def createFileArray(base: Path): Unit = { + createFile("f1", base, "line1\nline2\n") + createFile("f2", base, "line3\nline4\n") + createFile("f3", base, "line5\n") + } + + def cleanupFileArray(base: Path) = { + deleteFile(base.resolve("f1")) + deleteFile(base.resolve("f2")) + deleteFile(base.resolve("f3")) + } + + def deleteFile(path: Path) = Files.delete(path) + + private def createFile(name: String, dir: Path, contents: String) = { + dir.toFile.mkdirs() + write(dir.resolve(name).toFile, contents) + } + + private def write(file: File, contents: String) = { + val writer = new FileWriter(file) + writer.write(contents) + writer.flush() + writer.close() + file + } +} + +object SampleWdl { + object ThreeStep extends SampleWdl { + override def wdlSource(runtime: String = "") = sourceString() + + private val outputSectionPlaceholder = "OUTPUTSECTIONPLACEHOLDER" + + def sourceString(outputsSection: String = "") = { + val withPlaceholders = + """ + |task ps { + | command { + | ps + | } + | output { + | File procs = stdout() + | } + |} + | + |task cgrep { + | String pattern + | File in_file + | + | command { + | grep '${pattern}' ${in_file} | wc -l + | } + | output { + | Int count = read_int(stdout()) + | } + |} + | + |task wc { + | File in_file + | command { + | cat ${in_file} | wc -l + | } + | output { + | Int count = read_int(stdout()) + | } + |} + | + |workflow three_step { + | call ps + | call cgrep { + | input: in_file = ps.procs + | } + | call wc { + | input: in_file = ps.procs + | } + | """ + outputSectionPlaceholder + + """ + |} + | + """ + withPlaceholders.stripMargin.replace(outputSectionPlaceholder, outputsSection) + } + + val PatternKey = "three_step.cgrep.pattern" + override lazy val rawInputs = Map(PatternKey -> "...") + } + + object EmptyInvalid extends SampleWdl { + override def wdlSource(runtime: String = "") = "{}" + val rawInputs = Map.empty[String, Any] + } + + object EmptyWorkflow extends SampleWdl { + override def wdlSource(runtime: String = "") = "workflow empty_workflow {}" + val rawInputs = Map.empty[String, Any] + } + + object EmptyTask extends SampleWdl { + override def wdlSource(runtime: String = "") = "task empty_task { command { : } }" + val rawInputs = Map.empty[String, Any] + } +}