Skip to content

Commit

Permalink
igluctl: warn user if he tries to generate DDL not for 1-0-0 (close #221
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@oguzhanunlu
oguzhanunlu committed Jan 16, 2018
1 parent ff157f8 commit 5f13ad5
Show file tree
Hide file tree
Showing 2 changed files with 220 additions and 21 deletions.
103 changes: 84 additions & 19 deletions 0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/GenerateCommand.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import java.time.{ZoneOffset, ZonedDateTime}
import java.time.format.DateTimeFormatter

// Iglu Core
import com.snowplowanalytics.iglu.core.SchemaMap
import com.snowplowanalytics.iglu.core.{SchemaMap, SchemaVer}

// Schema DDL
import com.snowplowanalytics.iglu.schemaddl._
Expand Down Expand Up @@ -93,6 +93,8 @@ case class GenerateCommand(
// Parse Self-describing Schemas
val (schemaErrors, schemas) = splitValidations(files.map(_.extractSelfDescribingSchema))

val (schemaVerWarnings, schemaVerErrors) = validateSchemaVersions(schemas)

// Build table definitions from JSON Schemas
val validatedDdls = schemas.map(schema => selfDescSchemaToDdl(schema, dbSchemaStr).map(ddl => (schema.self, ddl)))
val (ddlErrors, ddlPairs) = splitValidations(validatedDdls)
Expand Down Expand Up @@ -120,7 +122,63 @@ case class GenerateCommand(
outputPair.map(_._1),
migrations,
outputPair.flatMap(_._2),
warnings = schemaErrors ++ ddlErrors ++ ddlWarnings)
warnings = schemaVerWarnings ++ schemaVerErrors ++ schemaErrors ++ ddlErrors ++ ddlWarnings)
}


/**
* Checks if there is any missing schema version in a directory of schemas
* or if a specific schema file doesn't have version 1-0-0
*
* @param schemas list of valid JSON Schemas including all Self-describing information
* @return (versionWarnings, versionErrors)
*/
private[ctl] def validateSchemaVersions(schemas: List[IgluSchema]): (List[String], List[String]) = {

def existMissingSchemaVersion(schemaMaps: List[SchemaMap]): Boolean = {
val numOfMaps = schemaMaps.length

if (numOfMaps == 1){
false
} else {
val prevModel = schemaMaps.head.version.model
val prevRevision = schemaMaps.head.version.revision
val prevAddition = schemaMaps.head.version.addition
val curModel = schemaMaps.tail.head.version.model
val curRevision = schemaMaps.tail.head.version.revision
val curAddition = schemaMaps.tail.head.version.addition

if (curModel == prevModel && curRevision == prevRevision && curAddition == prevAddition + 1 ||
curModel == prevModel && curRevision == prevRevision + 1 && curAddition == 0 ||
curModel == prevModel + 1 && curRevision == 0 && curAddition == 0)
existMissingSchemaVersion(schemaMaps.tail) else true
}
}

if (schemas.empty) (List.empty[String], List.empty[String])

if (input.isFile) {
val schemaVerWarning = schemas.head.self.version match {
case SchemaVer.Full(1, 0, 0) => List.empty[String]
case _ => List(s"Warning: File [${input.getAbsolutePath}] contains a schema whose version is NOT 1-0-0")
}
(schemaVerWarning, List.empty[String])
} else {
val schemaMapsGroupByName: Map[String, List[SchemaMap]] = schemas.map(schema => schema.self).groupBy(_.name)

val FirstVersionNotFoundErrors = for {
(k, v) <- schemaMapsGroupByName
if !v.exists(sm => sm.version == SchemaVer.Full(1, 0, 0)) && !force
} yield s"Error: Directory [${input.getAbsolutePath}] contains a schema with name [$k] without version 1-0-0. You can use --force to override."

val schemaVerGapErrors = for {
(k, v) <- schemaMapsGroupByName
sortedSchemaMaps = v.sortWith(_.version.asString < _.version.asString)
if sortedSchemaMaps.head.version == SchemaVer.Full(1, 0, 0) && existMissingSchemaVersion(sortedSchemaMaps) && !force
} yield s"Error: Directory [${input.getAbsolutePath}] contains a schema with name [$k] which has gaps between schema versions. You can use --force to override."

(List.empty[String], FirstVersionNotFoundErrors.toList ::: schemaVerGapErrors.toList)
}
}

/**
Expand Down Expand Up @@ -271,24 +329,31 @@ case class GenerateCommand(
* Output end result
*/
def outputResult(result: DdlOutput): Unit = {
result.ddls
.map(_.setBasePath("sql"))
.map(_.setBasePath(output.getAbsolutePath))
.map(_.write(force)).foreach(printMessage)

result.jsonPaths
.map(_.setBasePath("jsonpaths"))
.map(_.setBasePath(output.getAbsolutePath))
.map(_.write(force)).foreach(printMessage)

result.migrations
.map(_.setBasePath("sql"))
.map(_.setBasePath(output.getAbsolutePath))
.map(_.write(force)).foreach(printMessage)

result.warnings.foreach(printMessage)
val dirErr = result.warnings.filter(w => w.startsWith("Error: Directory"))
if (!dirErr.isEmpty) {
println(dirErr.head)
sys.exit(1)
} else {
result.ddls
.map(_.setBasePath("sql"))
.map(_.setBasePath(output.getAbsolutePath))
.map(_.write(force)).foreach(printMessage)

result.jsonPaths
.map(_.setBasePath("jsonpaths"))
.map(_.setBasePath(output.getAbsolutePath))
.map(_.write(force)).foreach(printMessage)

result.migrations
.map(_.setBasePath("sql"))
.map(_.setBasePath(output.getAbsolutePath))
.map(_.write(force)).foreach(printMessage)

result.warnings.foreach(printMessage)

if (result.warnings.exists(_.contains("Error"))) sys.exit(1)
}

if (result.warnings.exists(_.contains("Error"))) sys.exit(1)
}
}

Expand Down
138 changes: 136 additions & 2 deletions 0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/GenerateCommandSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,23 @@ import java.io.File

import com.snowplowanalytics.iglu.ctl.GenerateCommand.DdlOutput
import com.snowplowanalytics.iglu.ctl.FileUtils.TextFile

//
import com.snowplowanalytics.iglu.ctl.Utils.splitValidations
import com.snowplowanalytics.iglu.ctl.FileUtils.JsonFile

// Scalaz
import scalaz._
import Scalaz._


class GenerateCommandSpec extends Specification { def is = s2"""
DDL-generation command (ddl) specification
correctly convert com.amazon.aws.lambda/java_context_1 with default arguments $e1
correctly convert com.amazon.aws.lambda/java_context_1 with --raw --no-header --varchar 128 $e2
correctly convert com.amazon.aws.ec2/instance_identity_1 with --no-header --schema snowplow $e3
correctly produce JSONPaths file for com.amazon.aws.cloudfront/wd_access_log_1 $e4
output correct warnings for DDL-generation process $e5
warn about missing schema versions (addition) $e6
warn about missing 1-0-0 schema version $e7
"""

def e1 = {
Expand Down Expand Up @@ -613,4 +619,132 @@ class GenerateCommandSpec extends Specification { def is = s2"""
"Warning: in generated DDL [com.acme/other_context_1]: another_warning"
))
}

def e6 = {
val sourceSchema = parse(
"""
|{
| "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
| "description": "Schema for an example agency event",
| "self": {
| "vendor": "com.example-agency",
| "name": "cast",
| "format": "jsonschema",
| "version": "1-0-0"
| },
| "type": "object",
| "properties": {
| "name": {
| "type": "string"
| },
| "age": {
| "type": "number"
| }
| },
| "required":["name"]
|}
""".stripMargin
)
val sourceSchema2 = parse(
"""
|{
| "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
| "description": "Schema for an example agency event",
| "self": {
| "vendor": "com.example-agency",
| "name": "cast",
| "format": "jsonschema",
| "version": "1-0-3"
| },
| "type": "object",
| "properties": {
| "name": {
| "type": "string"
| },
| "surname": {
| "type": "string"
| },
| "age": {
| "type": "number"
| }
| },
| "required":["name", "surname"]
|}
""".stripMargin
)
val jsonFile = JsonFile(sourceSchema, new File("1-0-0"))
val jsonFile2 = JsonFile(sourceSchema2, new File("1-0-3"))
val stubFile: File = new File(".")
val command = GenerateCommand(stubFile, stubFile)
val (_, schemas) = splitValidations(List(jsonFile, jsonFile2).map(_.extractSelfDescribingSchema))
val (_, schemaVerErrors) = command.validateSchemaVersions(schemas)

schemaVerErrors must beEqualTo(List(
s"Error: Directory [${stubFile.getAbsolutePath}] contains a schema with name [cast] which has gaps between schema versions. You can use --force to override."
))
}

def e7 = {
val sourceSchema = parse(
"""
|{
| "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
| "description": "Schema for an example agency event",
| "self": {
| "vendor": "com.example-agency",
| "name": "cast",
| "format": "jsonschema",
| "version": "1-1-0"
| },
| "type": "object",
| "properties": {
| "name": {
| "type": "string"
| },
| "age": {
| "type": "number"
| }
| },
| "required":["name"]
|}
""".stripMargin
)
val sourceSchema2 = parse(
"""
|{
| "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
| "description": "Schema for an example agency event",
| "self": {
| "vendor": "com.example-agency",
| "name": "cast",
| "format": "jsonschema",
| "version": "1-1-1"
| },
| "type": "object",
| "properties": {
| "name": {
| "type": "string"
| },
| "surname": {
| "type": "string"
| },
| "age": {
| "type": "number"
| }
| },
| "required":["name", "surname"]
|}
""".stripMargin
)
val jsonFile = JsonFile(sourceSchema, new File("1-1-0"))
val jsonFile2 = JsonFile(sourceSchema2, new File("1-1-1"))
val stubFile: File = new File(".")
val command = GenerateCommand(stubFile, stubFile)
val (_, schemas) = splitValidations(List(jsonFile, jsonFile2).map(_.extractSelfDescribingSchema))
val (_, schemaVerErrors) = command.validateSchemaVersions(schemas)

schemaVerErrors must beEqualTo(List(
s"Error: Directory [${stubFile.getAbsolutePath}] contains a schema with name [cast] without version 1-0-0. You can use --force to override."
))
}
}

0 comments on commit 5f13ad5

Please sign in to comment.