W-16571553: merge avro feature to develop in amf

adrs/0014-avro-parsing.md → adrs/0014-avro-support.md

@@ -1,4 +1,4 @@
-# 13. AWS OAS parsing
+# 14. AVRO Support in AMF
 
 Date: 2024-07-02
 
@@ -10,8 +10,10 @@ Accepted
 
 ## Context
 
-Async 2.x supports AVRO Schemas and we currently don't. 
-We want to add support AVRO Schemas inside Async APIs and as a standalone documents.
+Async 2.x supports AVRO Schemas, and we currently don't. 
+We want to add support of AVRO Schemas:
+- inside Async APIs
+- as a standalone document
 
 We need to decide:
 - how are we going to map AVRO Schemas to the AMF Model
@@ -50,10 +52,32 @@ We've also added 3 AVRO-specific fields to the `AnyShape` Model via the `AvroFie
 - Aliases
 - Size
 
+### Where we support and DON'T support AVRO Schemas
+We Support AVRO Schemas (inline or inside a `$ref`):
+- as a standalone document or file 
+  - we encourage users to use the `.avsc` file type to indicate that's an avro file, for better suggestions and so on in the platform).
+- inside a message payload in an AsyncAPI
+  - the key `schemaFormat` MUST be declared and specify it's an AVRO payload
 
-For now only parsing is done 
+We don't support AVRO Schemas:
+- inside components --> schemas in an AsyncAPI
+  - because we can't determine if it's an AVRO Schema or any other schema
 
+### AVRO Validation
+We'll use the Apache official plugins for JVM and JS.
 
-## Consequences
 
-None so far.
+## Consequences / Constraints
+
+The validation plugins differ in interfaces and implementations, and each has some constraints:
+
+### JVM avro validation plugin
+- validation per se is not supported, we try to parse an avro schema and throw parsing results if there are any
+  - this means it's difficult to have location of where the error is thrown, we may give an approximate location from our end post-validation
+- when a validation is thrown, the rest of the file is not being searched for more validations
+  - this is particularly important in large avro schemas, where many errors can be found but only one is shown
+
+
+### Both JVM & JS validation plugins
+- `"default"` values are not being validated when the type is `bytes`, `map`, or `array`
+- the validator treats as invalid an empty array as the default value for arrays (`"default": []`) even though the [Avro Schema Specification](https://avro.apache.org/docs/1.12.0/specification) has some examples with it

amf-antlr-syntax/js/package.json

@@ -12,6 +12,7 @@
   },
   "dependencies": {
     "@aml-org/amf-antlr-parsers": "0.7.25",
-    "ajv": "6.12.6"
+    "ajv": "6.12.6",
+    "avro-js": "1.12.0"
   }
 }

amf-api-contract/js/package.json

@@ -12,6 +12,7 @@
   },
   "dependencies": {
     "@aml-org/amf-antlr-parsers": "0.7.25",
-    "ajv": "6.12.6"
+    "ajv": "6.12.6",
+    "avro-js": "^1.12.0"
   }
 }

amf-api-contract/shared/src/main/scala/amf/apicontract/client/scala/AMFConfiguration.scala

@@ -13,6 +13,7 @@ import amf.apicontract.internal.spec.avro.transformation.{
   AvroSchemaEditingPipeline,
   AvroSchemaTransformationPipeline
 }
+import amf.apicontract.internal.spec.avro.validation.AvroSchemaPayloadValidationPlugin
 import amf.apicontract.internal.spec.avro.{AvroParsePlugin, AvroRenderPlugin}
 import amf.apicontract.internal.spec.oas._
 import amf.apicontract.internal.spec.raml._
@@ -49,10 +50,12 @@ import amf.core.internal.validation.EffectiveValidations
 import amf.core.internal.validation.core.ValidationProfile
 import amf.shapes.client.scala.ShapesConfiguration
 import amf.shapes.client.scala.config.JsonSchemaConfiguration
-import amf.shapes.client.scala.plugin.JsonSchemaShapePayloadValidationPlugin
+import amf.shapes.client.scala.plugin.{AvroSchemaShapePayloadValidationPlugin, JsonSchemaShapePayloadValidationPlugin}
 import amf.shapes.internal.annotations.ShapeSerializableAnnotations
 import amf.shapes.internal.entities.ShapeEntities
 import amf.shapes.internal.spec.jsonschema.JsonSchemaParsePlugin
+import amf.shapes.internal.validation.model.ShapeEffectiveValidations.AvroSchemaEffectiveValidations
+import amf.shapes.internal.validation.model.ShapeValidationProfiles.AvroSchemaValidationProfile
 
 import scala.concurrent.Future
 
@@ -82,7 +85,8 @@ trait APIConfigurationBuilder {
       configuration.idAdopterProvider
     ).withPlugins(
       List(
-        JsonSchemaShapePayloadValidationPlugin
+        JsonSchemaShapePayloadValidationPlugin,
+        AvroSchemaShapePayloadValidationPlugin
       )
     ).withFallback(ApiContractFallbackPlugin())
     result
@@ -166,14 +170,21 @@ object RAMLConfiguration extends APIConfigurationBuilder {
 object AvroConfiguration extends APIConfigurationBuilder {
   def Avro(): AMFConfiguration = {
     common()
-      .withPlugins(List(AvroParsePlugin, AvroRenderPlugin)) // TODO: add validation profiles
+      .withPlugins(
+        List(
+          AvroParsePlugin,
+          AvroRenderPlugin,
+          AvroSchemaPayloadValidationPlugin()
+        )
+      )
       .withTransformationPipelines(
         List(
           AvroSchemaTransformationPipeline(),
           AvroSchemaEditingPipeline(),
           AvroSchemaCachePipeline()
         )
       )
+      .withValidationProfile(AvroSchemaValidationProfile, AvroSchemaEffectiveValidations)
   }
 }
 

amf-api-contract/shared/src/main/scala/amf/apicontract/internal/spec/async/emitters/domain/AsyncSchemaEmitter.scala

@@ -2,18 +2,20 @@ package amf.apicontract.internal.spec.async.emitters.domain
 
 import amf.apicontract.internal.spec.async.emitters.context.Async20SpecEmitterContext
 import amf.apicontract.internal.spec.async.parser.domain.AsyncSchemaFormats
+import amf.apicontract.internal.spec.avro.emitters.context.AvroShapeEmitterContext
+import amf.apicontract.internal.spec.avro.emitters.domain.AvroShapeEmitter
 import amf.apicontract.internal.spec.oas.emitter.context.{OasLikeShapeEmitterContextAdapter, OasLikeSpecEmitterContext}
 import amf.apicontract.internal.spec.raml.emitter
 import amf.apicontract.internal.spec.spec.toRaml
-import org.mulesoft.common.client.lexical.Position
 import amf.core.client.scala.model.document.BaseUnit
 import amf.core.client.scala.model.domain.Shape
-import amf.core.internal.render.BaseEmitters.pos
+import amf.core.internal.render.BaseEmitters.{pos, traverse}
 import amf.core.internal.render.SpecOrdering
 import amf.core.internal.render.emitters.EntryEmitter
 import amf.shapes.internal.spec.common.{AVROSchema, RAML10SchemaVersion, SchemaVersion}
 import amf.shapes.internal.spec.oas.emitter.OasTypePartEmitter
 import amf.shapes.internal.spec.raml.emitter.Raml10TypeEmitter
+import org.mulesoft.common.client.lexical.Position
 import org.yaml.model.YDocument.EntryBuilder
 
 case class AsyncSchemaEmitter(
@@ -59,9 +61,9 @@ case class AsyncSchemaEmitter(
       key,
       b => {
         val newCtx = new Async20SpecEmitterContext(spec.eh, config = spec.renderConfig, schemaVersion = schemaVersion)
-        // todo: call a specific AVRO Schema emitter (tbd in W-15633198)
-        OasTypePartEmitter(shape, ordering, references = references)(OasLikeShapeEmitterContextAdapter(newCtx))
-          .emit(b)
+        val entries =
+          AvroShapeEmitter(shape, ordering)(AvroShapeEmitterContext(newCtx.eh, newCtx.renderConfig)).entries()
+        b.obj((traverse(entries, _)))
       }
     )
   }