From bdb61fb59899513c65f40853e5510ae125b98754 Mon Sep 17 00:00:00 2001 From: Oguzhan Unlu Date: Thu, 18 Jan 2018 18:26:43 +0300 Subject: [PATCH] [WIP] igluctl: switch severity level to skip checks (close #232) --- .../ctl/Command.scala | 23 ++++++---- .../ctl/LintCommand.scala | 12 ++++- .../iglu/ctl/CommandSpec.scala | 4 +- .../jsonschema/SanityLinter.scala | 44 ++++++++++++------- .../jsonschema/SanityLinterSpec.scala | 20 ++++----- 5 files changed, 63 insertions(+), 40 deletions(-) diff --git a/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/Command.scala b/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/Command.scala index 3d788aae..86a25eb3 100644 --- a/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/Command.scala +++ b/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/Command.scala @@ -56,6 +56,7 @@ case class Command( // lint skipWarnings: Boolean = false, severityLevel: SeverityLevel = FirstLevel, + skipChecks: Option[String] = None, // s3 bucket: Option[String] = None, @@ -73,7 +74,7 @@ case class Command( case Some("static s3cp") => Some(S3cpCommand(input.get, bucket.get, s3path, accessKeyId, secretAccessKey, profile, region)) case Some("lint") => - Some(LintCommand(input.get, skipWarnings, severityLevel)) + Some(LintCommand(input.get, skipWarnings, severityLevel, skipChecks)) case _ => None } @@ -138,27 +139,27 @@ object Command { opt[File]("output") action { (x, c) => c.copy(output = Some(x)) } valueName "" - text "Directory to put generated data\t\tDefault: current dir", + text "Directory to put generated data\t\t\t\tDefault: current dir", opt[String]("dbschema") action { (x, c) => c.copy(schema = Some(x)) } valueName "" - text "Redshift schema name\t\t\t\tDefault: atomic", + text "Redshift schema name\t\t\t\t\t\tDefault: atomic", opt[String]("set-owner") action { (x, c) => c.copy(owner = Some(x)) } valueName "" - text "Redshift table owner\t\t\t\tDefault: None", + text "Redshift table owner\t\t\t\t\t\tDefault: None", opt[String]("db") action { (x, c) => c.copy(db = x) } valueName "" - text "DB to which we need to generate DDL\t\tDefault: redshift", + text "DB to which we need to generate DDL\t\t\t\tDefault: redshift", opt[Int]("varchar-size") action { (x, c) => c.copy(varcharSize = x) } valueName "" - text "Default size for varchar data type\t\tDefault: 4096", + text "Default size for varchar data type\t\t\t\tDefault: 4096", opt[Unit]("with-json-paths") action { (_, c) => c.copy(withJsonPaths = true) } @@ -225,7 +226,7 @@ object Command { opt[String]("s3path") action { (x, c) => c.copy(s3path = Some(x))} - text "Path in the bucket to upload Schemas\t\tDefault: bucket root", + text "Path in the bucket to upload Schemas\t\t\t\tDefault: bucket root", opt[String]("accessKeyId") optional() action { (x, c) => c.copy(accessKeyId = Some(x))} @@ -245,7 +246,7 @@ object Command { opt[String]("region") action { (x, c) => c.copy(region = Some(x))} valueName "" - text "AWS S3 region\t\t\t\tDefault: us-west-2\n", + text "AWS S3 region\t\t\t\t\t\tDefault: us-west-2\n", checkConfig { (c: Command) => (c.secretAccessKey, c.accessKeyId, c.profile) match { @@ -272,8 +273,12 @@ object Command { opt[SeverityLevel]("severityLevel") action { (x, c) => c.copy(severityLevel = x) } - text "Severity level\t\t\t\tDefault: 1" + text "Severity level\t\t\t\t\t\tDefault: 1", + opt[String]("skip-checks") + action { (x, c) => c.copy(skipChecks = Some(x)) } + valueName "" + text "Lint without provided linters, given comma separated\t\tDefault: None" ) } } diff --git a/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/LintCommand.scala b/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/LintCommand.scala index 88563232..24cdeebf 100644 --- a/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/LintCommand.scala +++ b/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/LintCommand.scala @@ -31,14 +31,14 @@ import com.github.fge.jsonschema.core.report.{ ListProcessingReport, ProcessingM // Schema DDL import com.snowplowanalytics.iglu.schemaddl.jsonschema.{ Schema, SanityLinter } -import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.SeverityLevel +import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.{ Linter, SeverityLevel } import com.snowplowanalytics.iglu.schemaddl.jsonschema.json4s.Json4sToSchema._ // This library import FileUtils.{ getJsonFilesStream, JsonFile, filterJsonSchemas } import Utils.{ extractSchema, splitValidations } -case class LintCommand(inputDir: File, skipWarnings: Boolean, severityLevel: SeverityLevel) extends Command.CtlCommand { +case class LintCommand(inputDir: File, skipWarnings: Boolean, severityLevel: SeverityLevel, skipChecks: Option[String]) extends Command.CtlCommand { import LintCommand._ /** @@ -72,6 +72,10 @@ case class LintCommand(inputDir: File, skipWarnings: Boolean, severityLevel: Sev if (!lintSchemaVer) { sys.exit(1) } else { +// val excludedLinters: List[Linter] = skipChecks match { +// case Some(linterStr) => validateSkippedLinters(severityLevel, linterStr) +// case None => List.empty[Linter] +// } val reports = jsons.map { file => val report = file.map(check) flattenReport(report) @@ -225,4 +229,8 @@ object LintCommand { case _ => true } else true + + def validateSkippedLinters(severityLevel: SeverityLevel, linters: String): List[Linter] = { + ??? + } } diff --git a/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/CommandSpec.scala b/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/CommandSpec.scala index 80640531..62841f63 100644 --- a/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/CommandSpec.scala +++ b/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/CommandSpec.scala @@ -33,10 +33,10 @@ class CommandSpec extends Specification { def is = s2""" def e1 = { val lint = Command .cliParser - .parse("lint . --severityLevel 2".split(" "), Command()) + .parse("lint . --severityLevel 2 --skip-checks lintUnknownFormats".split(" "), Command()) .flatMap(_.toCommand) - lint must beSome(LintCommand(new File("."), false, SecondLevel)) + lint must beSome(LintCommand(new File("."), false, SecondLevel, Some("lintUnknownFormats"))) } def e2 = { diff --git a/0-common/schema-ddl/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/SanityLinter.scala b/0-common/schema-ddl/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/SanityLinter.scala index eb3a4690..067fd1f3 100644 --- a/0-common/schema-ddl/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/SanityLinter.scala +++ b/0-common/schema-ddl/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/SanityLinter.scala @@ -106,18 +106,28 @@ object SanityLinter { } /** - * Main working function, traversing JSON Schema - * It lints all properties on current level, then tries to extract all - * subschemas from properties like `items`, `additionalItems` etc and - * recursively lint them as well - * - * @param schema parsed JSON AST - * @return non-empty list of summed failures (all, including nested) or - * unit in case of success - */ - def lint(schema: Schema, severityLevel: SeverityLevel, height: Int): LintSchema = { + * + * Main working function, traversing JSON Schema + * It lints all properties on current level, then tries to extract all + * subschemas from properties like `items`, `additionalItems` etc and + * recursively lint them as well + * + * @param schema parsed JSON AST + * @param severityLevel severity level + * @param height depth of linting + * @param excludedLinters list of linters to exclude + * @return non-empty list of summed failures (all, including nested) or + * unit in case of success + */ + def lint(schema: Schema, severityLevel: SeverityLevel, height: Int, excludedLinters: List[Linter]): LintSchema = { + + val linters = excludedLinters match { + case Nil => severityLevel.linters + case linterList => severityLevel.linters diff linterList + } + // Current level validations - val validations = severityLevel.linters.map(linter => linter(schema)) + val validations = linters.map(linter => linter(schema)) .foldMap(_.toValidationNel) val rootTypeCheck = @@ -140,30 +150,30 @@ object SanityLinter { val properties = schema.properties match { case Some(props) => - props.value.values.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1)) + props.value.values.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1, Nil)) case None => schemaSuccess } val patternProperties = schema.patternProperties match { case Some(PatternProperties(props)) => - props.values.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1)) + props.values.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1, Nil)) case _ => schemaSuccess } val additionalProperties = schema.additionalProperties match { - case Some(AdditionalPropertiesSchema(s)) => lint(s, severityLevel, height+1) + case Some(AdditionalPropertiesSchema(s)) => lint(s, severityLevel, height+1, Nil) case _ => schemaSuccess } val items = schema.items match { - case Some(ListItems(s)) => lint(s, severityLevel, height+1) + case Some(ListItems(s)) => lint(s, severityLevel, height+1, Nil) case Some(TupleItems(i)) => - i.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1)) + i.foldLeft(schemaSuccess)((a, s) => a |+| lint(s, severityLevel, height+1, Nil)) case None => schemaSuccess } val additionalItems = schema.additionalItems match { - case Some(AdditionalItemsSchema(s)) => lint(s, severityLevel, height+1) + case Some(AdditionalItemsSchema(s)) => lint(s, severityLevel, height+1, Nil) case _ => schemaSuccess } diff --git a/0-common/schema-ddl/src/test/scala/com/snowplowanalytics/iglu/schemaddl/jsonschema/SanityLinterSpec.scala b/0-common/schema-ddl/src/test/scala/com/snowplowanalytics/iglu/schemaddl/jsonschema/SanityLinterSpec.scala index ec996c25..f12d079b 100644 --- a/0-common/schema-ddl/src/test/scala/com/snowplowanalytics/iglu/schemaddl/jsonschema/SanityLinterSpec.scala +++ b/0-common/schema-ddl/src/test/scala/com/snowplowanalytics/iglu/schemaddl/jsonschema/SanityLinterSpec.scala @@ -47,7 +47,7 @@ class SanityLinterSpec extends Specification { def is = s2""" |} """.stripMargin)).get - SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("Properties [minLength] require string or absent type"))) + SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("Properties [minLength] require string or absent type"))) } def e2 = { @@ -78,7 +78,7 @@ class SanityLinterSpec extends Specification { def is = s2""" |} """.stripMargin)).get - SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("minimum property [5] is greater than maximum [0]"))) + SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("minimum property [5] is greater than maximum [0]"))) } def e3 = { @@ -94,7 +94,7 @@ class SanityLinterSpec extends Specification { def is = s2""" """.stripMargin )).get - SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("Properties [twoKey] is required, but not listed in properties"))) + SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("Properties [twoKey] is required, but not listed in properties"))) } def e4 = { @@ -128,7 +128,7 @@ class SanityLinterSpec extends Specification { def is = s2""" """.stripMargin )).get - SanityLinter.lint(schema, SanityLinter.SecondLevel, 0) must beEqualTo( + SanityLinter.lint(schema, SanityLinter.SecondLevel, 0, Nil) must beEqualTo( Failure(NonEmptyList( "String Schema doesn't contain maxLength nor enum properties nor appropriate format", "Numeric Schema doesn't contain minimum and maximum properties", @@ -173,7 +173,7 @@ class SanityLinterSpec extends Specification { def is = s2""" """.stripMargin )).get - SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo( + SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo( Failure(NonEmptyList( "Properties [maximum] require number, integer or absent type", "Properties [minimum] require number, integer or absent type" @@ -202,7 +202,7 @@ class SanityLinterSpec extends Specification { def is = s2""" """.stripMargin )).get - SanityLinter.lint(schema, SanityLinter.SecondLevel, 0) must beEqualTo( + SanityLinter.lint(schema, SanityLinter.SecondLevel, 0, Nil) must beEqualTo( Failure(NonEmptyList( "Schema doesn't begin with type object", "String Schema doesn't contain maxLength nor enum properties nor appropriate format" @@ -228,7 +228,7 @@ class SanityLinterSpec extends Specification { def is = s2""" """.stripMargin )).get - SanityLinter.lint(schema, SanityLinter.ThirdLevel, 0) must beEqualTo( + SanityLinter.lint(schema, SanityLinter.ThirdLevel, 0, Nil) must beEqualTo( Failure(NonEmptyList( "Object Schema doesn't contain description property", "It is recommended to express absence of property via nullable type", @@ -259,7 +259,7 @@ class SanityLinterSpec extends Specification { def is = s2""" """.stripMargin )).get - SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("Format [camelCase] is not supported. Available options are: date-time, date, email, hostname, ipv4, ipv6, uri"))) + SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("Format [camelCase] is not supported. Available options are: date-time, date, email, hostname, ipv4, ipv6, uri"))) } def e9 = { @@ -272,7 +272,7 @@ class SanityLinterSpec extends Specification { def is = s2""" |} """.stripMargin)).get - SanityLinter.lint(schema, SanityLinter.FirstLevel, 0) must beEqualTo(Failure(NonEmptyList("maxLength [65536] is greater than Redshift VARCHAR(max), 65535"))) + SanityLinter.lint(schema, SanityLinter.FirstLevel, 0, Nil) must beEqualTo(Failure(NonEmptyList("maxLength [65536] is greater than Redshift VARCHAR(max), 65535"))) } def e10 = { @@ -317,7 +317,7 @@ def e10 = { """.stripMargin )).get - SanityLinter.lint(schema, SanityLinter.ThirdLevel, 0) must beEqualTo( + SanityLinter.lint(schema, SanityLinter.ThirdLevel, 0, Nil) must beEqualTo( Failure(NonEmptyList( "It is recommended to express absence of property via nullable type", "String Schema doesn't contain description property",