[WIP] igluctl: switch severity level to skip checks (close #232)

snowplow · Jan 19, 2018 · 97b6088 · 97b6088
1 parent 7d79ff0
commit 97b6088
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 40 deletions.
diff --git a/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/Command.scala b/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/Command.scala
@@ -56,6 +56,7 @@ case class Command(
   // lint
   skipWarnings:    Boolean         = false,
   severityLevel:   SeverityLevel   = FirstLevel,
+  skipChecks:      Option[String]  = None,
 
   // s3
   bucket:          Option[String]  = None,
@@ -73,7 +74,7 @@ case class Command(
     case Some("static s3cp") =>
       Some(S3cpCommand(input.get, bucket.get, s3path, accessKeyId, secretAccessKey, profile, region))
     case Some("lint") =>
-      Some(LintCommand(input.get, skipWarnings, severityLevel))
+      Some(LintCommand(input.get, skipWarnings, severityLevel, skipChecks))
     case _ =>
       None
   }
@@ -138,27 +139,27 @@ object Command {
             opt[File]("output")
               action { (x, c) => c.copy(output = Some(x)) }
               valueName "<path>"
-              text "Directory to put generated data\t\tDefault: current dir",
+              text "Directory to put generated data\t\t\t\tDefault: current dir",
 
             opt[String]("dbschema")
               action { (x, c) => c.copy(schema = Some(x)) }
               valueName "<name>"
-              text "Redshift schema name\t\t\t\tDefault: atomic",
+              text "Redshift schema name\t\t\t\t\t\tDefault: atomic",
 
             opt[String]("set-owner")
               action { (x, c) => c.copy(owner = Some(x)) }
               valueName "<owner>"
-              text "Redshift table owner\t\t\t\tDefault: None",
+              text "Redshift table owner\t\t\t\t\t\tDefault: None",
 
             opt[String]("db")
               action { (x, c) => c.copy(db = x) }
               valueName "<name>"
-              text "DB to which we need to generate DDL\t\tDefault: redshift",
+              text "DB to which we need to generate DDL\t\t\t\tDefault: redshift",
 
             opt[Int]("varchar-size")
               action { (x, c) => c.copy(varcharSize = x) }
               valueName "<n>"
-              text "Default size for varchar data type\t\tDefault: 4096",
+              text "Default size for varchar data type\t\t\t\tDefault: 4096",
 
             opt[Unit]("with-json-paths")
               action { (_, c) => c.copy(withJsonPaths = true) }
@@ -225,7 +226,7 @@ object Command {
 
             opt[String]("s3path")
               action { (x, c) => c.copy(s3path = Some(x))}
-              text "Path in the bucket to upload Schemas\t\tDefault: bucket root",
+              text "Path in the bucket to upload Schemas\t\t\t\tDefault: bucket root",
 
             opt[String]("accessKeyId") optional()
               action { (x, c) => c.copy(accessKeyId = Some(x))}
@@ -245,7 +246,7 @@ object Command {
             opt[String]("region")
               action { (x, c) => c.copy(region = Some(x))}
               valueName "<name>"
-              text "AWS S3 region\t\t\t\tDefault: us-west-2\n",
+              text "AWS S3 region\t\t\t\t\t\tDefault: us-west-2\n",
 
             checkConfig { (c: Command) =>
               (c.secretAccessKey, c.accessKeyId, c.profile) match {
@@ -272,8 +273,12 @@ object Command {
 
         opt[SeverityLevel]("severityLevel")
           action { (x, c) => c.copy(severityLevel = x) }
-          text "Severity level\t\t\t\tDefault: 1"
+          text "Severity level\t\t\t\t\t\tDefault: 1",
 
+        opt[String]("skip-checks")
+          action { (x, c) => c.copy(skipChecks = Some(x)) }
+          valueName "<l1,l2...>"
+          text "Lint without provided linters, given comma separated\t\tDefault: None"
       )
   }
 }

diff --git a/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/LintCommand.scala b/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/LintCommand.scala
@@ -31,14 +31,14 @@ import com.github.fge.jsonschema.core.report.{ ListProcessingReport, ProcessingM
 
 // Schema DDL
 import com.snowplowanalytics.iglu.schemaddl.jsonschema.{ Schema, SanityLinter }
-import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.SeverityLevel
+import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.{ Linter, SeverityLevel }
 import com.snowplowanalytics.iglu.schemaddl.jsonschema.json4s.Json4sToSchema._
 
 // This library
 import FileUtils.{ getJsonFilesStream, JsonFile, filterJsonSchemas }
 import Utils.{ extractSchema, splitValidations }
 
-case class LintCommand(inputDir: File, skipWarnings: Boolean, severityLevel: SeverityLevel) extends Command.CtlCommand {
+case class LintCommand(inputDir: File, skipWarnings: Boolean, severityLevel: SeverityLevel, skipChecks: Option[String]) extends Command.CtlCommand {
   import LintCommand._
 
   /**
@@ -75,6 +75,10 @@ case class LintCommand(inputDir: File, skipWarnings: Boolean, severityLevel: Sev
     if (!lintSchemaVer) {
       sys.exit(1)
     } else {
+//      val excludedLinters: List[Linter] = skipChecks match {
+//        case Some(linterStr) => validateSkippedLinters(severityLevel, linterStr)
+//        case None => List.empty[Linter]
+//      }
       val reports = jsons.map { file =>
         val report = file.map(check)
         flattenReport(report)
@@ -228,4 +232,8 @@ object LintCommand {
       case _ => true
     }
     else true
+
+  def validateSkippedLinters(severityLevel: SeverityLevel, linters: String): List[Linter] = {
+    ???
+  }
 }
diff --git a/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/CommandSpec.scala b/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/CommandSpec.scala
@@ -33,10 +33,10 @@ class CommandSpec extends Specification { def is = s2"""
   def e1 = {
     val lint = Command
       .cliParser
-      .parse("lint . --severityLevel 2".split(" "), Command())
+      .parse("lint . --severityLevel 2 --skip-checks lintUnknownFormats".split(" "), Command())
       .flatMap(_.toCommand)
 
-    lint must beSome(LintCommand(new File("."), false, SecondLevel))
+    lint must beSome(LintCommand(new File("."), false, SecondLevel, Some("lintUnknownFormats")))
   }
 
   def e2 = {

diff --git a/...ema-ddl/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/SanityLinter.scala b/...ema-ddl/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/SanityLinter.scala
@@ -106,18 +106,28 @@ object SanityLinter {
   }
 
   /**
-   * Main working function, traversing JSON Schema
-   * It lints all properties on current level, then tries to extract all
-   * subschemas from properties like `items`, `additionalItems` etc and
-   * recursively lint them as well
-   *
-   * @param schema parsed JSON AST
-   * @return non-empty list of summed failures (all, including nested) or
-   *         unit in case of success
-   */
-  def lint(schema: Schema, severityLevel: SeverityLevel, height: Int): LintSchema = {
+    *
+    * Main working function, traversing JSON Schema
+    * It lints all properties on current level, then tries to extract all
+    * subschemas from properties like `items`, `additionalItems` etc and
+    * recursively lint them as well
+    *
+    * @param schema parsed JSON AST
+    * @param severityLevel severity level
+    * @param height depth of linting
+    * @param excludedLinters list of linters to exclude
+    * @return non-empty list of summed failures (all, including nested) or
+    *         unit in case of success
+    */
+  def lint(schema: Schema, severityLevel: SeverityLevel, height: Int, excludedLinters: List[Linter]): LintSchema = {
+
+    val linters = excludedLinters match {
+      case Nil => severityLevel.linters
+      case linterList => severityLevel.linters diff linterList
+    }
+
     // Current level validations
-    val validations = severityLevel.linters.map(linter => linter(schema))
+    val validations = linters.map(linter => linter(schema))
       .foldMap(_.toValidationNel)
 
     val rootTypeCheck =
@@ -140,30 +150,30 @@ object SanityLinter {
 
     val properties = schema.properties match {
       case Some(props) =>
-        props.value.values.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1))
+        props.value.values.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1, Nil))
       case None => schemaSuccess
     }
 
     val patternProperties = schema.patternProperties match {
       case Some(PatternProperties(props)) =>
-        props.values.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1))
+        props.values.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1, Nil))
       case _ => schemaSuccess
     }
 
     val additionalProperties = schema.additionalProperties match {
-      case Some(AdditionalPropertiesSchema(s)) => lint(s, severityLevel, height+1)
+      case Some(AdditionalPropertiesSchema(s)) => lint(s, severityLevel, height+1, Nil)
       case _ => schemaSuccess
     }
 
     val items = schema.items match {
-      case Some(ListItems(s)) => lint(s, severityLevel, height+1)
+      case Some(ListItems(s)) => lint(s, severityLevel, height+1, Nil)
       case Some(TupleItems(i)) =>
-        i.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1))
+        i.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1, Nil))
       case None => schemaSuccess
     }
 
     val additionalItems = schema.additionalItems match {
-      case Some(AdditionalItemsSchema(s)) => lint(s, severityLevel, height+1)
+      case Some(AdditionalItemsSchema(s)) => lint(s, severityLevel, height+1, Nil)
       case _ => schemaSuccess
     }
 

diff --git a/...ddl/src/test/scala/com/snowplowanalytics/iglu/schemaddl/jsonschema/SanityLinterSpec.scala b/...ddl/src/test/scala/com/snowplowanalytics/iglu/schemaddl/jsonschema/SanityLinterSpec.scala
@@ -47,7 +47,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
         |}
       """.stripMargin)).get
 
-    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("Properties [minLength] require string or absent type")))
+    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("Properties [minLength] require string or absent type")))
   }
 
   def e2 = {
@@ -78,7 +78,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
         |}
       """.stripMargin)).get
 
-    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("minimum property [5] is greater than maximum [0]")))
+    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("minimum property [5] is greater than maximum [0]")))
   }
 
   def e3 = {
@@ -94,7 +94,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
       """.stripMargin
     )).get
 
-    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("Properties [twoKey] is required, but not listed in properties")))
+    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("Properties [twoKey] is required, but not listed in properties")))
   }
 
   def e4 = {
@@ -128,7 +128,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
       """.stripMargin
     )).get
 
-    SanityLinter.lint(schema, SanityLinter.SecondLevel, 0) must beEqualTo(
+    SanityLinter.lint(schema, SanityLinter.SecondLevel, 0, Nil) must beEqualTo(
       Failure(NonEmptyList(
         "String Schema doesn't contain maxLength nor enum properties nor appropriate format",
         "Numeric Schema doesn't contain minimum and maximum properties",
@@ -173,7 +173,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
       """.stripMargin
     )).get
 
-    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(
+    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(
       Failure(NonEmptyList(
         "Properties [maximum] require number, integer or absent type",
         "Properties [minimum] require number, integer or absent type"
@@ -202,7 +202,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
       """.stripMargin
     )).get
 
-    SanityLinter.lint(schema, SanityLinter.SecondLevel, 0) must beEqualTo(
+    SanityLinter.lint(schema, SanityLinter.SecondLevel, 0, Nil) must beEqualTo(
       Failure(NonEmptyList(
         "Schema doesn't begin with type object",
         "String Schema doesn't contain maxLength nor enum properties nor appropriate format"
@@ -228,7 +228,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
       """.stripMargin
     )).get
 
-    SanityLinter.lint(schema, SanityLinter.ThirdLevel, 0) must beEqualTo(
+    SanityLinter.lint(schema, SanityLinter.ThirdLevel, 0, Nil) must beEqualTo(
       Failure(NonEmptyList(
         "Object Schema doesn't contain description property",
         "It is recommended to express absence of property via nullable type",
@@ -259,7 +259,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
       """.stripMargin
     )).get
 
-    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("Format [camelCase] is not supported. Available options are: date-time, date, email, hostname, ipv4, ipv6, uri")))
+    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("Format [camelCase] is not supported. Available options are: date-time, date, email, hostname, ipv4, ipv6, uri")))
   }
 
   def e9 = {
@@ -272,7 +272,7 @@ class SanityLinterSpec extends Specification { def is = s2"""
         |}
       """.stripMargin)).get
 
-    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("maxLength [65536] is greater than Redshift VARCHAR(max), 65535")))
+    SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("maxLength [65536] is greater than Redshift VARCHAR(max), 65535")))
   }
 
 def e10 = {
@@ -317,7 +317,7 @@ def e10 = {
       """.stripMargin
     )).get
 
-    SanityLinter.lint(schema, SanityLinter.ThirdLevel, 0) must beEqualTo(
+    SanityLinter.lint(schema, SanityLinter.ThirdLevel, 0, Nil) must beEqualTo(
       Failure(NonEmptyList(
         "It is recommended to express absence of property via nullable type",
         "String Schema doesn't contain description property",