-
Notifications
You must be signed in to change notification settings - Fork 39
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
2 changed files
with
163 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,6 +10,8 @@ | |
| */ | ||
| package com.snowplowanalytics.snowplow.enrich.common.enrichments | ||
|
|
||
| import java.time.Instant | ||
|
|
||
| import org.apache.commons.codec.digest.DigestUtils | ||
|
|
||
| import org.specs2.mutable.Specification | ||
|
|
@@ -1273,7 +1275,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE | |
| def expectedDerivedContexts(enriched: EnrichedEvent): Boolean = { | ||
| val emailSentSDJ = SelfDescribingData.parse[Json](jparse(emailSent).toOption.get).toOption.get | ||
| SpecHelpers.listContexts(enriched.derived_contexts) match { | ||
| case List(`emailSentSDJ`, SelfDescribingData(Failure.`failureSchemaKey`, feJson)) | ||
| case List(SelfDescribingData(Failure.`failureSchemaKey`, feJson), `emailSentSDJ`) | ||
| if feJson.field("failureType") == "ValidationError".asJson && | ||
| feJson.field("errors") == Json.arr( | ||
| Json.obj( | ||
|
|
@@ -1362,7 +1364,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE | |
| def expectedDerivedContexts(enriched: EnrichedEvent): Boolean = { | ||
| val emailSentSDJ = SelfDescribingData.parse[Json](jparse(emailSent).toOption.get).toOption.get | ||
| SpecHelpers.listContexts(enriched.derived_contexts) match { | ||
| case List(`emailSentSDJ`, SelfDescribingData(Failure.`failureSchemaKey`, feJson)) | ||
| case List(SelfDescribingData(Failure.`failureSchemaKey`, feJson), `emailSentSDJ`) | ||
| if feJson.field("failureType") == "ValidationError".asJson && | ||
| feJson.field("errors") == Json.arr( | ||
| Json.obj( | ||
|
|
@@ -1461,7 +1463,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE | |
| def expectedDerivedContexts(enriched: EnrichedEvent): Boolean = { | ||
| val emailSentSDJ = SelfDescribingData.parse[Json](jparse(emailSent).toOption.get).toOption.get | ||
| SpecHelpers.listContexts(enriched.derived_contexts) match { | ||
| case List(`emailSentSDJ`, SelfDescribingData(Failure.`failureSchemaKey`, feJson)) | ||
| case List(SelfDescribingData(Failure.`failureSchemaKey`, feJson), `emailSentSDJ`) | ||
| if feJson.field("failureType") == "ValidationError".asJson && | ||
| feJson.field("errors") == Json.arr( | ||
| Json.obj( | ||
|
|
@@ -1543,8 +1545,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE | |
| def expectedDerivedContexts(enriched: EnrichedEvent): Boolean = | ||
| SpecHelpers.listContexts(enriched.derived_contexts) match { | ||
| case List( | ||
| SelfDescribingData(SchemaKey("nl.basjes", "yauaa_context", "jsonschema", _), _), | ||
| SelfDescribingData(Failure.`failureSchemaKey`, feJson) | ||
| SelfDescribingData(Failure.`failureSchemaKey`, feJson), | ||
| SelfDescribingData(SchemaKey("nl.basjes", "yauaa_context", "jsonschema", _), _) | ||
| ) | ||
| if feJson.field("failureType") == "EnrichmentError: Javascript enrichment".asJson && | ||
| feJson.field("errors") == Json.arr( | ||
|
|
@@ -1704,8 +1706,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE | |
| def expectedDerivedContexts(enriched: EnrichedEvent): Boolean = | ||
| SpecHelpers.listContexts(enriched.derived_contexts) match { | ||
| case List( | ||
| SelfDescribingData(SchemaKey("nl.basjes", "yauaa_context", "jsonschema", _), _), | ||
| SelfDescribingData(Failure.`failureSchemaKey`, feJson) | ||
| SelfDescribingData(Failure.`failureSchemaKey`, feJson), | ||
| SelfDescribingData(SchemaKey("nl.basjes", "yauaa_context", "jsonschema", _), _) | ||
| ) | ||
| if feJson.field("failureType") == "ValidationError".asJson && | ||
| feJson.field("errors") == Json.arr( | ||
|
|
@@ -1750,6 +1752,92 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE | |
| case other => ko(s"[$other] is not a SchemaViolations bad row and an enriched event without the faulty enrichment context") | ||
| } | ||
| } | ||
|
|
||
| "return a bad row that contains validation errors only from ue if there is validation error in both ue and derived contexts when emitIncomplete is set to true" >> { | ||
| val invalidContext = | ||
| """ | ||
| { | ||
| "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0", | ||
| "data": { | ||
| "foo": "[email protected]", | ||
| "emailAddress2": "[email protected]" | ||
| } | ||
| }""" | ||
| val invalidUe = | ||
| """{ | ||
| "schema":"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-1", | ||
| "data": { | ||
| "unallowedAdditionalField": "[email protected]" | ||
| } | ||
| }""" | ||
| val script = | ||
| s""" | ||
| function process(event) { | ||
| return [ | ||
| $invalidContext | ||
| ]; | ||
| }""" | ||
| val schemaKey = SchemaKey( | ||
| "com.snowplowanalytics.snowplow", | ||
| "javascript_script_config", | ||
| "jsonschema", | ||
| SchemaVer.Full(1, 0, 0) | ||
| ) | ||
| val enrichmentReg = EnrichmentRegistry[IO]( | ||
| javascriptScript = List( | ||
| JavascriptScriptEnrichment(schemaKey, script) | ||
| ) | ||
| ) | ||
| val parameters = Map( | ||
| "e" -> "pp", | ||
| "tv" -> "js-0.13.1", | ||
| "p" -> "web", | ||
| "ue_pr" -> | ||
| s""" | ||
| { | ||
| "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", | ||
| "data": $invalidUe | ||
| }""" | ||
| ).toOpt | ||
| val rawEvent = RawEvent(api, parameters, None, source, context) | ||
| val enriched = EnrichmentManager.enrichEvent[IO]( | ||
| enrichmentReg, | ||
| client, | ||
| processor, | ||
| timestamp, | ||
| rawEvent, | ||
| AcceptInvalid.featureFlags, | ||
| IO.unit, | ||
| SpecHelpers.registryLookup, | ||
| atomicFieldLimits, | ||
| emitIncomplete = true | ||
| ) | ||
| def expectedDerivedContexts(enriched: EnrichedEvent): Boolean = | ||
| SpecHelpers.listContextsSchemas(enriched.derived_contexts).count(_ == Failure.failureSchemaKey) == 2 | ||
|
|
||
| def expectedBadRow(badRow: BadRow): Boolean = | ||
| badRow match { | ||
| case BadRow.SchemaViolations( | ||
| _, | ||
| BadRowFailure.SchemaViolations( | ||
| _, | ||
| NonEmptyList(FailureDetails.SchemaViolation.IgluError(`clientSessionSchema`, _), Nil) | ||
| ), | ||
| _ | ||
| ) => | ||
| true | ||
| case _ => false | ||
| } | ||
|
|
||
| enriched.value.map { | ||
| case Ior.Both(badRow, enriched) | ||
| if Option(enriched.unstruct_event).isEmpty && | ||
| expectedDerivedContexts(enriched) && | ||
| expectedBadRow(badRow) => | ||
| ok | ||
| case other => ko(s"[$other] is not expected one") | ||
| } | ||
| } | ||
| } | ||
|
|
||
| "getCrossDomain" should { | ||
|
|
@@ -2330,6 +2418,50 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE | |
| } | ||
| } | ||
| } | ||
|
|
||
| "setDerivedContexts" should { | ||
| val sv = Failure.SchemaViolation( | ||
| schemaViolation = FailureDetails.SchemaViolation.NotJson("testField", "testValue".some, "testError"), | ||
| source = "testSource", | ||
| data = Json.obj("testKey" := "testValue") | ||
| ) | ||
| val ef = Failure.EnrichmentFailure( | ||
| FailureDetails.EnrichmentFailure( | ||
| None, | ||
| FailureDetails.EnrichmentFailureMessage.Simple("testError") | ||
| ) | ||
| ) | ||
| val emailSentSDJ = SelfDescribingData.parse[Json](jparse(emailSent).toOption.get).toOption.get | ||
| val timestamp = Instant.now() | ||
| "set derived contexts correctly if enrichment result is Ior.Left" >> { | ||
| val enriched = new EnrichedEvent() | ||
| val enrichmentResult = Ior.Left(NonEmptyList.of(NonEmptyList.of(sv, ef), NonEmptyList.of(sv, ef))) | ||
| EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, timestamp, processor) | ||
| val schemas = SpecHelpers.listContextsSchemas(enriched.derived_contexts) | ||
| schemas.size must beEqualTo(4) | ||
| forall(schemas)(s => s must beEqualTo(Failure.failureSchemaKey)) | ||
| } | ||
| "set derived contexts correctly if enrichment result is Ior.Right" >> { | ||
| val enriched = new EnrichedEvent() | ||
| val enrichmentResult = Ior.Right(List(emailSentSDJ, emailSentSDJ)) | ||
| EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, timestamp, processor) | ||
| val schemas = SpecHelpers.listContextsSchemas(enriched.derived_contexts) | ||
| schemas.size must beEqualTo(2) | ||
| forall(schemas)(s => s must beEqualTo(emailSentSchema)) | ||
| } | ||
| "set derived contexts correctly if enrichment result is Ior.Both" >> { | ||
| val enriched = new EnrichedEvent() | ||
| val enrichmentResult = Ior.Both( | ||
| NonEmptyList.of(NonEmptyList.of(sv, ef), NonEmptyList.of(sv, ef)), | ||
| List(emailSentSDJ, emailSentSDJ) | ||
| ) | ||
| EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, timestamp, processor) | ||
| val schemas = SpecHelpers.listContextsSchemas(enriched.derived_contexts) | ||
| schemas.size must beEqualTo(6) | ||
| schemas.count(_ == Failure.failureSchemaKey) must beEqualTo(4) | ||
| schemas.count(_ == emailSentSchema) must beEqualTo(2) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| object EnrichmentManagerSpec { | ||
|
|
||