From 4eb4f865ab46062d161044574d619d09758b7c14 Mon Sep 17 00:00:00 2001 From: Oguzhan Unlu Date: Tue, 26 Dec 2017 20:46:08 +0300 Subject: [PATCH] igluctl: warn user if he tries to generate DDL not for 1-0-0 (close #221) --- .../ctl/GenerateCommand.scala | 105 ++++++++++--- .../iglu/ctl/GenerateCommandSpec.scala | 138 +++++++++++++++++- 2 files changed, 222 insertions(+), 21 deletions(-) diff --git a/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/GenerateCommand.scala b/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/GenerateCommand.scala index 37c478fa..895e3d52 100644 --- a/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/GenerateCommand.scala +++ b/0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/GenerateCommand.scala @@ -18,7 +18,7 @@ import java.time.{ZoneOffset, ZonedDateTime} import java.time.format.DateTimeFormatter // Iglu Core -import com.snowplowanalytics.iglu.core.SchemaMap +import com.snowplowanalytics.iglu.core.{SchemaMap, SchemaVer} // Schema DDL import com.snowplowanalytics.iglu.schemaddl._ @@ -93,6 +93,8 @@ case class GenerateCommand( // Parse Self-describing Schemas val (schemaErrors, schemas) = splitValidations(files.map(_.extractSelfDescribingSchema)) + val (schemaVerWarnings, schemaVerErrors) = validateSchemaVersions(schemas) + // Build table definitions from JSON Schemas val validatedDdls = schemas.map(schema => selfDescSchemaToDdl(schema, dbSchemaStr).map(ddl => (schema.self, ddl))) val (ddlErrors, ddlPairs) = splitValidations(validatedDdls) @@ -120,7 +122,65 @@ case class GenerateCommand( outputPair.map(_._1), migrations, outputPair.flatMap(_._2), - warnings = schemaErrors ++ ddlErrors ++ ddlWarnings) + warnings = schemaVerWarnings ++ schemaVerErrors ++ schemaErrors ++ ddlErrors ++ ddlWarnings) + } + + + /** + * Checks if there is any missing schema version in a directory of schemas + * or if a specific schema file doesn't have version 1-0-0 + * + * @param schemas list of valid JSON Schemas including all Self-describing information + * @return (versionWarnings, versionErrors) + */ + private[ctl] def validateSchemaVersions(schemas: List[IgluSchema]): (List[String], List[String]) = { + + def existMissingSchemaVersion(schemaMaps: List[SchemaMap]): Boolean = { + val numOfMaps = schemaMaps.length + + if (numOfMaps == 1){ + false + } else { + val prevModel = schemaMaps.head.version.model + val prevRevision = schemaMaps.head.version.revision + val prevAddition = schemaMaps.head.version.addition + val curModel = schemaMaps.tail.head.version.model + val curRevision = schemaMaps.tail.head.version.revision + val curAddition = schemaMaps.tail.head.version.addition + + if (curModel == prevModel && curRevision == prevRevision && curAddition == prevAddition + 1 || + curModel == prevModel && curRevision == prevRevision + 1 && curAddition == 0 || + curModel == prevModel + 1 && curRevision == 0 && curAddition == 0) + existMissingSchemaVersion(schemaMaps.tail) else true + } + } + + if (schemas.empty) { + (List.empty[String], List.empty[String]) + } else { + if (input.isFile) { + val schemaVerWarning = schemas.head.self.version match { + case SchemaVer.Full(1, 0, 0) => List.empty[String] + case _ => List(s"Warning: File [${input.getAbsolutePath}] contains a schema whose version is NOT 1-0-0") + } + (schemaVerWarning, List.empty[String]) + } else { + val schemaMapsGroupByName: Map[String, List[SchemaMap]] = schemas.map(schema => schema.self).groupBy(_.name) + + val FirstVersionNotFoundErrors = for { + (k, v) <- schemaMapsGroupByName + if !v.exists(sm => sm.version == SchemaVer.Full(1, 0, 0)) && !force + } yield s"Error: Directory [${input.getAbsolutePath}] contains a schema with name [$k] without version 1-0-0. You can use --force to override." + + val schemaVerGapErrors = for { + (k, v) <- schemaMapsGroupByName + sortedSchemaMaps = v.sortWith(_.version.asString < _.version.asString) + if sortedSchemaMaps.head.version == SchemaVer.Full(1, 0, 0) && existMissingSchemaVersion(sortedSchemaMaps) && !force + } yield s"Error: Directory [${input.getAbsolutePath}] contains a schema with name [$k] which has gaps between schema versions. You can use --force to override." + + (List.empty[String], FirstVersionNotFoundErrors.toList ::: schemaVerGapErrors.toList) + } + } } /** @@ -271,24 +331,31 @@ case class GenerateCommand( * Output end result */ def outputResult(result: DdlOutput): Unit = { - result.ddls - .map(_.setBasePath("sql")) - .map(_.setBasePath(output.getAbsolutePath)) - .map(_.write(force)).foreach(printMessage) - - result.jsonPaths - .map(_.setBasePath("jsonpaths")) - .map(_.setBasePath(output.getAbsolutePath)) - .map(_.write(force)).foreach(printMessage) - - result.migrations - .map(_.setBasePath("sql")) - .map(_.setBasePath(output.getAbsolutePath)) - .map(_.write(force)).foreach(printMessage) - - result.warnings.foreach(printMessage) + val dirErr = result.warnings.filter(w => w.startsWith("Error: Directory")) + if (!dirErr.isEmpty) { + println(dirErr.head) + sys.exit(1) + } else { + result.ddls + .map(_.setBasePath("sql")) + .map(_.setBasePath(output.getAbsolutePath)) + .map(_.write(force)).foreach(printMessage) + + result.jsonPaths + .map(_.setBasePath("jsonpaths")) + .map(_.setBasePath(output.getAbsolutePath)) + .map(_.write(force)).foreach(printMessage) + + result.migrations + .map(_.setBasePath("sql")) + .map(_.setBasePath(output.getAbsolutePath)) + .map(_.write(force)).foreach(printMessage) + + result.warnings.foreach(printMessage) + + if (result.warnings.exists(_.contains("Error"))) sys.exit(1) + } - if (result.warnings.exists(_.contains("Error"))) sys.exit(1) } } diff --git a/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/GenerateCommandSpec.scala b/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/GenerateCommandSpec.scala index 078727b6..513d7eba 100644 --- a/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/GenerateCommandSpec.scala +++ b/0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/GenerateCommandSpec.scala @@ -18,10 +18,14 @@ import java.io.File import com.snowplowanalytics.iglu.ctl.GenerateCommand.DdlOutput import com.snowplowanalytics.iglu.ctl.FileUtils.TextFile - -// +import com.snowplowanalytics.iglu.ctl.Utils.splitValidations import com.snowplowanalytics.iglu.ctl.FileUtils.JsonFile +// Scalaz +import scalaz._ +import Scalaz._ + + class GenerateCommandSpec extends Specification { def is = s2""" DDL-generation command (ddl) specification correctly convert com.amazon.aws.lambda/java_context_1 with default arguments $e1 @@ -29,6 +33,8 @@ class GenerateCommandSpec extends Specification { def is = s2""" correctly convert com.amazon.aws.ec2/instance_identity_1 with --no-header --schema snowplow $e3 correctly produce JSONPaths file for com.amazon.aws.cloudfront/wd_access_log_1 $e4 output correct warnings for DDL-generation process $e5 + warn about missing schema versions (addition) $e6 + warn about missing 1-0-0 schema version $e7 """ def e1 = { @@ -613,4 +619,132 @@ class GenerateCommandSpec extends Specification { def is = s2""" "Warning: in generated DDL [com.acme/other_context_1]: another_warning" )) } + + def e6 = { + val sourceSchema = parse( + """ + |{ + | "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + | "description": "Schema for an example agency event", + | "self": { + | "vendor": "com.example-agency", + | "name": "cast", + | "format": "jsonschema", + | "version": "1-0-0" + | }, + | "type": "object", + | "properties": { + | "name": { + | "type": "string" + | }, + | "age": { + | "type": "number" + | } + | }, + | "required":["name"] + |} + """.stripMargin + ) + val sourceSchema2 = parse( + """ + |{ + | "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + | "description": "Schema for an example agency event", + | "self": { + | "vendor": "com.example-agency", + | "name": "cast", + | "format": "jsonschema", + | "version": "1-0-3" + | }, + | "type": "object", + | "properties": { + | "name": { + | "type": "string" + | }, + | "surname": { + | "type": "string" + | }, + | "age": { + | "type": "number" + | } + | }, + | "required":["name", "surname"] + |} + """.stripMargin + ) + val jsonFile = JsonFile(sourceSchema, new File("1-0-0")) + val jsonFile2 = JsonFile(sourceSchema2, new File("1-0-3")) + val stubFile: File = new File(".") + val command = GenerateCommand(stubFile, stubFile) + val (_, schemas) = splitValidations(List(jsonFile, jsonFile2).map(_.extractSelfDescribingSchema)) + val (_, schemaVerErrors) = command.validateSchemaVersions(schemas) + + schemaVerErrors must beEqualTo(List( + s"Error: Directory [${stubFile.getAbsolutePath}] contains a schema with name [cast] which has gaps between schema versions. You can use --force to override." + )) + } + + def e7 = { + val sourceSchema = parse( + """ + |{ + | "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + | "description": "Schema for an example agency event", + | "self": { + | "vendor": "com.example-agency", + | "name": "cast", + | "format": "jsonschema", + | "version": "1-1-0" + | }, + | "type": "object", + | "properties": { + | "name": { + | "type": "string" + | }, + | "age": { + | "type": "number" + | } + | }, + | "required":["name"] + |} + """.stripMargin + ) + val sourceSchema2 = parse( + """ + |{ + | "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + | "description": "Schema for an example agency event", + | "self": { + | "vendor": "com.example-agency", + | "name": "cast", + | "format": "jsonschema", + | "version": "1-1-1" + | }, + | "type": "object", + | "properties": { + | "name": { + | "type": "string" + | }, + | "surname": { + | "type": "string" + | }, + | "age": { + | "type": "number" + | } + | }, + | "required":["name", "surname"] + |} + """.stripMargin + ) + val jsonFile = JsonFile(sourceSchema, new File("1-1-0")) + val jsonFile2 = JsonFile(sourceSchema2, new File("1-1-1")) + val stubFile: File = new File(".") + val command = GenerateCommand(stubFile, stubFile) + val (_, schemas) = splitValidations(List(jsonFile, jsonFile2).map(_.extractSelfDescribingSchema)) + val (_, schemaVerErrors) = command.validateSchemaVersions(schemas) + + schemaVerErrors must beEqualTo(List( + s"Error: Directory [${stubFile.getAbsolutePath}] contains a schema with name [cast] without version 1-0-0. You can use --force to override." + )) + } }