Skip to content

Commit

Permalink
[WIP] igluctl: switch severity level to skip checks (close #232)
Browse files Browse the repository at this point in the history
  • Loading branch information
@oguzhanunlu
oguzhanunlu committed Jan 23, 2018
1 parent 7d79ff0 commit 8f8f4c2
Show file tree
Hide file tree
Showing 7 changed files with 258 additions and 177 deletions.
43 changes: 22 additions & 21 deletions 0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/Command.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@ import java.util.UUID
// scopt
import scopt.OptionParser

// Schema DDL
import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.{ SeverityLevel, FirstLevel, SecondLevel, ThirdLevel }

// This library
import PushCommand._
import Utils.LinterList

// Schema DDL
import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.allLinters

/**
* Common command container
Expand Down Expand Up @@ -55,7 +56,7 @@ case class Command(

// lint
skipWarnings: Boolean = false,
severityLevel: SeverityLevel = FirstLevel,
linters: LinterList = allLinters.values.toList,

// s3
bucket: Option[String] = None,
Expand All @@ -73,7 +74,7 @@ case class Command(
case Some("static s3cp") =>
Some(S3cpCommand(input.get, bucket.get, s3path, accessKeyId, secretAccessKey, profile, region))
case Some("lint") =>
Some(LintCommand(input.get, skipWarnings, severityLevel))
Some(LintCommand(input.get, skipWarnings, linters))
case _ =>
None
}
Expand All @@ -91,11 +92,11 @@ object Command {
}
}

implicit val severityLevelRead: scopt.Read[SeverityLevel] = scopt.Read.reads {
case "1" => FirstLevel
case "2" => SecondLevel
case "3" => ThirdLevel
case l => throw new IllegalArgumentException(s"Error: $l is invalid severity level")
implicit val lintersRead: scopt.Read[LinterList] = scopt.Read.reads { s =>
LintCommand.validateSkippedLinters(s) match {
case Left(err) => throw new IllegalArgumentException(err)
case Right(linters) => linters
}
}

private def subcommand(sub: String)(unit: Unit, root: Command): Command =
Expand Down Expand Up @@ -138,27 +139,27 @@ object Command {
opt[File]("output")
action { (x, c) => c.copy(output = Some(x)) }
valueName "<path>"
text "Directory to put generated data\t\tDefault: current dir",
text "Directory to put generated data\t\t\t\tDefault: current dir",

opt[String]("dbschema")
action { (x, c) => c.copy(schema = Some(x)) }
valueName "<name>"
text "Redshift schema name\t\t\t\tDefault: atomic",
text "Redshift schema name\t\t\t\t\t\tDefault: atomic",

opt[String]("set-owner")
action { (x, c) => c.copy(owner = Some(x)) }
valueName "<owner>"
text "Redshift table owner\t\t\t\tDefault: None",
text "Redshift table owner\t\t\t\t\t\tDefault: None",

opt[String]("db")
action { (x, c) => c.copy(db = x) }
valueName "<name>"
text "DB to which we need to generate DDL\t\tDefault: redshift",
text "DB to which we need to generate DDL\t\t\t\tDefault: redshift",

opt[Int]("varchar-size")
action { (x, c) => c.copy(varcharSize = x) }
valueName "<n>"
text "Default size for varchar data type\t\tDefault: 4096",
text "Default size for varchar data type\t\t\t\tDefault: 4096",

opt[Unit]("with-json-paths")
action { (_, c) => c.copy(withJsonPaths = true) }
Expand Down Expand Up @@ -225,7 +226,7 @@ object Command {

opt[String]("s3path")
action { (x, c) => c.copy(s3path = Some(x))}
text "Path in the bucket to upload Schemas\t\tDefault: bucket root",
text "Path in the bucket to upload Schemas\t\t\t\tDefault: bucket root",

opt[String]("accessKeyId") optional()
action { (x, c) => c.copy(accessKeyId = Some(x))}
Expand All @@ -245,7 +246,7 @@ object Command {
opt[String]("region")
action { (x, c) => c.copy(region = Some(x))}
valueName "<name>"
text "AWS S3 region\t\t\t\tDefault: us-west-2\n",
text "AWS S3 region\t\t\t\t\t\tDefault: us-west-2\n",

checkConfig { (c: Command) =>
(c.secretAccessKey, c.accessKeyId, c.profile) match {
Expand All @@ -270,10 +271,10 @@ object Command {
action { (_, c) => c.copy(skipWarnings = true) }
text "Don't output messages with log level less than ERROR",

opt[SeverityLevel]("severityLevel")
action { (x, c) => c.copy(severityLevel = x) }
text "Severity level\t\t\t\tDefault: 1"

opt[LinterList]("skip-checks")
action { (x, c) => c.copy(linters = x) }
valueName "<l1,l2..>"
text "Lint without provided linters, given comma separated\t\tDefault: None"
)
}
}
Expand Down
32 changes: 28 additions & 4 deletions 0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/LintCommand.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,13 @@ import com.github.fge.jsonschema.core.report.{ ListProcessingReport, ProcessingM

// Schema DDL
import com.snowplowanalytics.iglu.schemaddl.jsonschema.{ Schema, SanityLinter }
import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.SeverityLevel
import com.snowplowanalytics.iglu.schemaddl.jsonschema.json4s.Json4sToSchema._

// This library
import FileUtils.{ getJsonFilesStream, JsonFile, filterJsonSchemas }
import Utils.{ extractSchema, splitValidations }
import Utils.{ extractSchema, splitValidations, LinterList }

case class LintCommand(inputDir: File, skipWarnings: Boolean, severityLevel: SeverityLevel) extends Command.CtlCommand {
case class LintCommand(inputDir: File, skipWarnings: Boolean, linters: LinterList) extends Command.CtlCommand {
import LintCommand._

/**
Expand Down Expand Up @@ -93,7 +92,7 @@ case class LintCommand(inputDir: File, skipWarnings: Boolean, severityLevel: Sev
val pathCheck = extractSchema(jsonFile).map(_ => ()).validation.toValidationNel
val syntaxCheck = validateSchema(jsonFile.content, skipWarnings)

val lintCheck = Schema.parse(jsonFile.content).map { schema => SanityLinter.lint(schema, severityLevel, 0) }
val lintCheck = Schema.parse(jsonFile.content).map { schema => SanityLinter.lint(schema, 0, linters) }

val fullValidation = syntaxCheck |+| pathCheck |+| lintCheck.getOrElse("Doesn't contain JSON Schema".failureNel)

Expand Down Expand Up @@ -228,4 +227,29 @@ object LintCommand {
case _ => true
}
else true

def validateSkippedLinters(skipChecks: String): Either[String, LinterList] = {
val skippedLinters = skipChecks.split(",")

val linterValidationErrors = for {
sl <- skippedLinters
linter = SanityLinter.allLinters.get(sl) match {
case Some(l) => l
case None => Nil
}
if linter == Nil
} yield s"Unknown linter $sl"

if (linterValidationErrors.nonEmpty) {
Left(linterValidationErrors.mkString("\n"))
} else {
val lintersToUse = skippedLinters.foldLeft(Right(SanityLinter.allLinters.values.toList)) { (linters, cur) =>
(linters, SanityLinter.allLinters.get(cur)) match {
case (Right(linters), Some(l)) => Right(linters.diff(List(l)))
case (Right(linters), None) => Right(linters)
}
}
lintersToUse
}
}
}
3 changes: 3 additions & 0 deletions 0-common/igluctl/src/main/scala/com.snowplowanalytics.iglu/ctl/Utils.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,12 @@ import com.snowplowanalytics.iglu.schemaddl.{ IgluSchema, RevisionGroup, ModelGr

// This library
import FileUtils.{ JsonFile, splitPath }
import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.Linter

object Utils {

type LinterList = List[Linter]

type Failing[+A] = String \/ A

/**
Expand Down
6 changes: 3 additions & 3 deletions 0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/CommandSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import java.io.File
import java.util.UUID

// Schema DDL
import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter._
import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.allLinters

// specs2
import org.specs2.Specification
Expand All @@ -33,10 +33,10 @@ class CommandSpec extends Specification { def is = s2"""
def e1 = {
val lint = Command
.cliParser
.parse("lint . --severityLevel 2".split(" "), Command())
.parse("lint .".split(" "), Command())
.flatMap(_.toCommand)

lint must beSome(LintCommand(new File("."), false, SecondLevel))
lint must beSome(LintCommand(new File("."), false, allLinters.values.toList))
}

def e2 = {
Expand Down
133 changes: 64 additions & 69 deletions 0-common/igluctl/src/test/scala/com/snowplowanalytics/iglu/ctl/GenerateCommandSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,29 +43,29 @@ class GenerateCommandSpec extends Specification { def is = s2"""
"""|CREATE SCHEMA IF NOT EXISTS atomic;
|
|CREATE TABLE IF NOT EXISTS atomic.com_amazon_aws_lambda_java_context_1 (
| "schema_vendor" VARCHAR(128) ENCODE RUNLENGTH NOT NULL,
| "schema_name" VARCHAR(128) ENCODE RUNLENGTH NOT NULL,
| "schema_format" VARCHAR(128) ENCODE RUNLENGTH NOT NULL,
| "schema_version" VARCHAR(128) ENCODE RUNLENGTH NOT NULL,
| "root_id" CHAR(36) ENCODE RAW NOT NULL,
| "root_tstamp" TIMESTAMP ENCODE LZO NOT NULL,
| "ref_root" VARCHAR(255) ENCODE RUNLENGTH NOT NULL,
| "ref_tree" VARCHAR(1500) ENCODE RUNLENGTH NOT NULL,
| "ref_parent" VARCHAR(255) ENCODE RUNLENGTH NOT NULL,
| "aws_request_id" VARCHAR(4096) ENCODE LZO,
| "client_context.client.app_package_name" VARCHAR(4096) ENCODE LZO,
| "client_context.client.app_title" VARCHAR(4096) ENCODE LZO,
| "client_context.client.app_version_code" VARCHAR(4096) ENCODE LZO,
| "client_context.client.app_version_name" VARCHAR(4096) ENCODE LZO,
| "client_context.custom" VARCHAR(4096) ENCODE LZO,
| "client_context.environment" VARCHAR(4096) ENCODE LZO,
| "function_name" VARCHAR(4096) ENCODE LZO,
| "identity.identity_id" VARCHAR(4096) ENCODE LZO,
| "identity.identity_pool_id" VARCHAR(4096) ENCODE LZO,
| "log_group_name" VARCHAR(4096) ENCODE LZO,
| "log_stream_name" VARCHAR(4096) ENCODE LZO,
| "memory_limit_in_mb" BIGINT ENCODE LZO,
| "remaining_time_millis" BIGINT ENCODE LZO,
| "schema_vendor" VARCHAR(128) ENCODE ZSTD NOT NULL,
| "schema_name" VARCHAR(128) ENCODE ZSTD NOT NULL,
| "schema_format" VARCHAR(128) ENCODE ZSTD NOT NULL,
| "schema_version" VARCHAR(128) ENCODE ZSTD NOT NULL,
| "root_id" CHAR(36) ENCODE RAW NOT NULL,
| "root_tstamp" TIMESTAMP ENCODE ZSTD NOT NULL,
| "ref_root" VARCHAR(255) ENCODE ZSTD NOT NULL,
| "ref_tree" VARCHAR(1500) ENCODE ZSTD NOT NULL,
| "ref_parent" VARCHAR(255) ENCODE ZSTD NOT NULL,
| "aws_request_id" VARCHAR(4096) ENCODE ZSTD,
| "client_context.client.app_package_name" VARCHAR(4096) ENCODE ZSTD,
| "client_context.client.app_title" VARCHAR(4096) ENCODE ZSTD,
| "client_context.client.app_version_code" VARCHAR(4096) ENCODE ZSTD,
| "client_context.client.app_version_name" VARCHAR(4096) ENCODE ZSTD,
| "client_context.custom" VARCHAR(4096) ENCODE ZSTD,
| "client_context.environment" VARCHAR(4096) ENCODE ZSTD,
| "function_name" VARCHAR(4096) ENCODE ZSTD,
| "identity.identity_id" VARCHAR(4096) ENCODE ZSTD,
| "identity.identity_pool_id" VARCHAR(4096) ENCODE ZSTD,
| "log_group_name" VARCHAR(4096) ENCODE ZSTD,
| "log_stream_name" VARCHAR(4096) ENCODE ZSTD,
| "memory_limit_in_mb" BIGINT ENCODE ZSTD,
| "remaining_time_millis" BIGINT ENCODE ZSTD,
| FOREIGN KEY (root_id) REFERENCES atomic.events(event_id)
|)
|DISTSTYLE KEY
Expand Down Expand Up @@ -166,39 +166,33 @@ class GenerateCommandSpec extends Specification { def is = s2"""

val jsonFile = JsonFile(sourceSchema, new File("1-0-0"))
val stubFile: File = new File(".")
val command = GenerateCommand(stubFile, stubFile)
val command = GenerateCommand(stubFile, stubFile, noHeader = true)

val output = command.transformSelfDescribing(List(jsonFile))

val expected = GenerateCommand.DdlOutput(List(TextFile(new File("com.amazon.aws.lambda/java_context_1.sql"), resultContent)))

def dropHeader(o: DdlOutput): DdlOutput = {
val textFile = o.ddls.head.file
val shortDdl = o.ddls.head.content.split("\n").toList.drop(4).mkString("\n")
val shortTextFiles = List(TextFile(textFile, shortDdl))
o.copy(ddls = shortTextFiles)
}
output must beEqualTo(expected)

dropHeader(output) must beEqualTo(expected)
}

def e2 = {
val resultContent =
"""|CREATE TABLE IF NOT EXISTS java_context (
| "aws_request_id" VARCHAR(128) ENCODE LZO,
| "client_context.client.app_package_name" VARCHAR(128) ENCODE LZO,
| "client_context.client.app_title" VARCHAR(128) ENCODE LZO,
| "client_context.client.app_version_code" VARCHAR(128) ENCODE LZO,
| "client_context.client.app_version_name" VARCHAR(128) ENCODE LZO,
| "client_context.custom" VARCHAR(128) ENCODE LZO,
| "client_context.environment" VARCHAR(128) ENCODE LZO,
| "function_name" VARCHAR(128) ENCODE LZO,
| "identity.identity_id" VARCHAR(128) ENCODE LZO,
| "identity.identity_pool_id" VARCHAR(128) ENCODE LZO,
| "log_group_name" VARCHAR(128) ENCODE LZO,
| "log_stream_name" VARCHAR(128) ENCODE LZO,
| "memory_limit_in_mb" BIGINT ENCODE LZO,
| "remaining_time_millis" BIGINT ENCODE LZO
| "aws_request_id" VARCHAR(128) ENCODE ZSTD,
| "client_context.client.app_package_name" VARCHAR(128) ENCODE ZSTD,
| "client_context.client.app_title" VARCHAR(128) ENCODE ZSTD,
| "client_context.client.app_version_code" VARCHAR(128) ENCODE ZSTD,
| "client_context.client.app_version_name" VARCHAR(128) ENCODE ZSTD,
| "client_context.custom" VARCHAR(128) ENCODE ZSTD,
| "client_context.environment" VARCHAR(128) ENCODE ZSTD,
| "function_name" VARCHAR(128) ENCODE ZSTD,
| "identity.identity_id" VARCHAR(128) ENCODE ZSTD,
| "identity.identity_pool_id" VARCHAR(128) ENCODE ZSTD,
| "log_group_name" VARCHAR(128) ENCODE ZSTD,
| "log_stream_name" VARCHAR(128) ENCODE ZSTD,
| "memory_limit_in_mb" BIGINT ENCODE ZSTD,
| "remaining_time_millis" BIGINT ENCODE ZSTD
|);
|
|COMMENT ON TABLE java_context IS 'Source: java-context.json';""".stripMargin
Expand Down Expand Up @@ -302,29 +296,29 @@ class GenerateCommandSpec extends Specification { def is = s2"""
"""|CREATE SCHEMA IF NOT EXISTS snowplow;
|
|CREATE TABLE IF NOT EXISTS snowplow.com_amazon_aws_ec2_instance_identity_document_1 (
| "schema_vendor" VARCHAR(128) ENCODE RUNLENGTH NOT NULL,
| "schema_name" VARCHAR(128) ENCODE RUNLENGTH NOT NULL,
| "schema_format" VARCHAR(128) ENCODE RUNLENGTH NOT NULL,
| "schema_version" VARCHAR(128) ENCODE RUNLENGTH NOT NULL,
| "root_id" CHAR(36) ENCODE RAW NOT NULL,
| "root_tstamp" TIMESTAMP ENCODE LZO NOT NULL,
| "ref_root" VARCHAR(255) ENCODE RUNLENGTH NOT NULL,
| "ref_tree" VARCHAR(1500) ENCODE RUNLENGTH NOT NULL,
| "ref_parent" VARCHAR(255) ENCODE RUNLENGTH NOT NULL,
| "account_id" VARCHAR(4096) ENCODE LZO,
| "architecture" VARCHAR(4096) ENCODE LZO,
| "availability_zone" VARCHAR(4096) ENCODE LZO,
| "billing_products" VARCHAR(5000) ENCODE LZO,
| "devpay_product_codes" VARCHAR(5000) ENCODE LZO,
| "image_id" CHAR(12) ENCODE LZO,
| "instance_id" VARCHAR(19) ENCODE LZO,
| "instance_type" VARCHAR(4096) ENCODE LZO,
| "kernel_id" CHAR(12) ENCODE LZO,
| "pending_time" TIMESTAMP ENCODE LZO,
| "private_ip" VARCHAR(15) ENCODE LZO,
| "ramdisk_id" CHAR(12) ENCODE LZO,
| "region" VARCHAR(4096) ENCODE LZO,
| "version" VARCHAR(4096) ENCODE LZO,
| "schema_vendor" VARCHAR(128) ENCODE ZSTD NOT NULL,
| "schema_name" VARCHAR(128) ENCODE ZSTD NOT NULL,
| "schema_format" VARCHAR(128) ENCODE ZSTD NOT NULL,
| "schema_version" VARCHAR(128) ENCODE ZSTD NOT NULL,
| "root_id" CHAR(36) ENCODE RAW NOT NULL,
| "root_tstamp" TIMESTAMP ENCODE ZSTD NOT NULL,
| "ref_root" VARCHAR(255) ENCODE ZSTD NOT NULL,
| "ref_tree" VARCHAR(1500) ENCODE ZSTD NOT NULL,
| "ref_parent" VARCHAR(255) ENCODE ZSTD NOT NULL,
| "account_id" VARCHAR(4096) ENCODE ZSTD,
| "architecture" VARCHAR(4096) ENCODE ZSTD,
| "availability_zone" VARCHAR(4096) ENCODE ZSTD,
| "billing_products" VARCHAR(5000) ENCODE ZSTD,
| "devpay_product_codes" VARCHAR(5000) ENCODE ZSTD,
| "image_id" CHAR(12) ENCODE ZSTD,
| "instance_id" VARCHAR(19) ENCODE ZSTD,
| "instance_type" VARCHAR(4096) ENCODE ZSTD,
| "kernel_id" CHAR(12) ENCODE ZSTD,
| "pending_time" TIMESTAMP ENCODE ZSTD,
| "private_ip" VARCHAR(15) ENCODE ZSTD,
| "ramdisk_id" CHAR(12) ENCODE ZSTD,
| "region" VARCHAR(4096) ENCODE ZSTD,
| "version" VARCHAR(4096) ENCODE ZSTD,
| FOREIGN KEY (root_id) REFERENCES snowplow.events(event_id)
|)
|DISTSTYLE KEY
Expand Down Expand Up @@ -770,7 +764,7 @@ class GenerateCommandSpec extends Specification { def is = s2"""

val resultContent =
"""|CREATE TABLE IF NOT EXISTS 1_0_0 (
| "name" VARCHAR(4096) ENCODE LZO NOT NULL,
| "name" VARCHAR(4096) ENCODE ZSTD NOT NULL,
| "age" DOUBLE PRECISION ENCODE RAW
|);
|
Expand All @@ -788,5 +782,6 @@ class GenerateCommandSpec extends Specification { def is = s2"""
)

ddl must beEqualTo(expected)

}
}
Loading

0 comments on commit 8f8f4c2

Please sign in to comment.