From 79196db04aae704986c2ac4430e695278357f564 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Thu, 24 Feb 2022 16:54:48 -0800 Subject: [PATCH 1/5] rename to serializer --- .vscode/cspell.json | 3 +- sdk/eventhub/azure-eventhub/CHANGELOG.md | 6 + sdk/eventhub/azure-eventhub/README.md | 6 +- .../azure-eventhub/azure/eventhub/_version.py | 2 +- .../CHANGELOG.md | 66 +++ .../LICENSE | 21 + .../MANIFEST.in | 6 + .../README.md | 329 ++++++++++++ .../azure/__init__.py | 26 + .../azure/schemaregistry/__init__.py | 26 + .../schemaregistry/serializer/__init__.py | 26 + .../serializer/avroserializer/__init__.py | 37 ++ .../_abstract_avro_serializer.py | 68 +++ .../avroserializer/_apache_avro_serializer.py | 100 ++++ .../serializer/avroserializer/_constants.py | 40 ++ .../avroserializer/_message_protocol.py | 39 ++ .../_schema_registry_avro_serializer.py | 285 ++++++++++ .../serializer/avroserializer/_version.py | 27 + .../serializer/avroserializer/aio/__init__.py | 30 ++ .../avroserializer/aio/_async_lru.py | 230 ++++++++ .../_schema_registry_avro_serializer_async.py | 277 ++++++++++ .../serializer/avroserializer/exceptions.py | 69 +++ .../dev_requirements.txt | 4 + .../mypy.ini | 6 + .../samples/README.md | 65 +++ .../eventhub_receive_integration_async.py | 73 +++ .../eventhub_send_integration_async.py | 79 +++ ...nd_deserialize_event_data_message_async.py | 107 ++++ ...ize_and_deserialize_with_metadata_async.py | 97 ++++ .../serialize_with_callback_async.py | 88 +++ .../eventhub_receive_integration.py | 65 +++ .../sync_samples/eventhub_send_integration.py | 71 +++ ...lize_and_deserialize_event_data_message.py | 101 ++++ ...serialize_and_deserialize_with_metadata.py | 91 ++++ .../sync_samples/serialize_with_callback.py | 83 +++ .../sdk_packaging.toml | 2 + .../setup.py | 72 +++ ...serializer_deserialize_readers_schema.yaml | 92 ++++ ...serializer_with_auto_register_schemas.yaml | 140 +++++ ...ializer_without_auto_register_schemas.yaml | 140 +++++ ...zer.test_parse_error_schema_as_record.yaml | 94 ++++ ...vro_serializer.test_parse_record_name.yaml | 185 +++++++ ...o_serializer.test_serialize_primitive.yaml | 48 ++ ...avro_serializer.test_serialize_record.yaml | 96 ++++ ...serializer_deserialize_readers_schema.yaml | 63 +++ ...serializer_with_auto_register_schemas.yaml | 99 ++++ ...ializer_without_auto_register_schemas.yaml | 99 ++++ ...ync.test_parse_error_schema_as_record.yaml | 65 +++ ...rializer_async.test_parse_record_name.yaml | 127 +++++ ...alizer_async.test_serialize_primitive.yaml | 34 ++ ...erializer_async.test_serialize_record.yaml | 67 +++ .../tests/test_avro_serializer.py | 500 ++++++++++++++++++ .../tests/test_avro_serializer_async.py | 476 +++++++++++++++++ 53 files changed, 5042 insertions(+), 6 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/LICENSE create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_abstract_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_apache_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_constants.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_message_protocol.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_version.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/__init__.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_async_lru.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/exceptions.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/mypy.ini create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_receive_integration_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_send_integration_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_event_data_message_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_with_metadata_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_with_callback_async.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_receive_integration.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_send_integration.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_event_data_message.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_with_metadata.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_with_callback.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/sdk_packaging.toml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer_async.py diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 8228c0b94d4e..455f4437a5ac 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -85,9 +85,8 @@ "sdk/metricsadvisor/azure-ai-metricsadvisor/**", "sdk/purview/azure-purview-catalog/**", "sdk/remoterendering/azure-mixedreality-remoterendering/**", - "sdk/schemaregistry/ci.yml", "sdk/schemaregistry/azure-schemaregistry/**", - "sdk/schemaregistry/azure-schemaregistry-avroencoder/**", + "sdk/schemaregistry/azure-schemaregistry-avroserializer/**", "sdk/servicefabric/azure-servicefabric/**", "sdk/search/azure-search-documents/**", "sdk/storage/azure-storage-blob-changefeed/**", diff --git a/sdk/eventhub/azure-eventhub/CHANGELOG.md b/sdk/eventhub/azure-eventhub/CHANGELOG.md index b5eb6a93fac1..731c33a67947 100644 --- a/sdk/eventhub/azure-eventhub/CHANGELOG.md +++ b/sdk/eventhub/azure-eventhub/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 5.9.0b2 (Unreleased) + +### Other Changes + +- The TODO: for interoperability with the Schema Registry Avro Serializer library. + ## 5.9.0b1 (2022-02-09) ### Features Added diff --git a/sdk/eventhub/azure-eventhub/README.md b/sdk/eventhub/azure-eventhub/README.md index 1d8e6ddebe94..a1552d2d8583 100644 --- a/sdk/eventhub/azure-eventhub/README.md +++ b/sdk/eventhub/azure-eventhub/README.md @@ -435,10 +435,10 @@ Please take a look at the [samples](https://github.com/Azure/azure-sdk-for-pytho Reference documentation is available [here](https://azuresdkdocs.blob.core.windows.net/$web/python/azure-eventhub/latest/azure.eventhub.html). -### Schema Registry and Avro Encoder +### Schema Registry and Avro Serializer The EventHubs SDK integrates nicely with the [Schema Registry][schemaregistry_service] service and [Avro][avro]. -For more information, please refer to [Schema Registry SDK][schemaregistry_repo] and [Schema Registry Avro Encoder SDK][schemaregistry_avroencoder_repo]. +For more information, please refer to [Schema Registry SDK][schemaregistry_repo] and [Schema Registry Avro Serializer SDK][schemaregistry_avroserializer_repo]. ### Building uAMQP wheel from source @@ -467,6 +467,6 @@ For more information see the [Code of Conduct FAQ](https://opensource.microsoft. [api_reference]: https://docs.microsoft.com/python/api/overview/azure/eventhub-readme [schemaregistry_service]: https://aka.ms/schemaregistry [schemaregistry_repo]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry -[schemaregistry_avroencoder_repo]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder +[schemaregistry_avroserializer_repo]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer ![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python/sdk/eventhub/azure-eventhub/README.png) diff --git a/sdk/eventhub/azure-eventhub/azure/eventhub/_version.py b/sdk/eventhub/azure-eventhub/azure/eventhub/_version.py index 71bbbef9cb3a..56ad08bfb3ed 100644 --- a/sdk/eventhub/azure-eventhub/azure/eventhub/_version.py +++ b/sdk/eventhub/azure-eventhub/azure/eventhub/_version.py @@ -3,4 +3,4 @@ # Licensed under the MIT License. # ------------------------------------ -VERSION = "5.9.0b1" +VERSION = "5.9.0b2" diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md new file mode 100644 index 000000000000..9db5c4105efc --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md @@ -0,0 +1,66 @@ +# Release History + +## 1.0.0b5 (Unreleased) + +### Features Added + +- APIs have been updated to allow for serializing directly to and deserializing from message type objects, where the data value is the Avro serialized payload. +- The content type of the message will hold the schema ID and record format indicator. + +### Other Changes + +- This beta release will be backward compatible for decoding data that was encoded with the `AvroSerializer` in `azure-schemaregistry-avroserializer==1.0.0b4`. +- The `serialize` and `deserialize` methods on `AvroSerializer` support the following message models: + - `azure.eventhub.EventData` in `azure-eventhub==5.9.0b2` +- Between this beta release and `azure-schemaregistry-avroserializer==1.0.0b4`, `azure-schemaregistry-avroencoder==1.0.0b1` was released before settling on `serialize/deserialize` terminology. + +## 1.0.0b4 (2021-11-11) + +### Features Added + +- Async version of `AvroSerializer` has been added under `azure.schemaregistry.serializer.avroserializer.aio`. +- Depends on `azure-schemaregistry>=1.0.0,<2.0.0`. + +### Breaking Changes + +- `SchemaParseError`, `SchemaSerializationError`, and `SchemaDeserializationError` have been introduced under `azure.schemaregistry.serializer.avroserializer.exceptions` and will be raised for corresponding operations. + - `SchemaParseError` and `SchemaSerializationError` may be raised for errors when calling `serialize` on `AvroSerializer`. + - `SchemaParseError` and `SchemaDeserializationError` may be raised for errors when calling `deserialize` on `AvroSerializer`. + +## 1.0.0b3 (2021-10-06) + +### Features Added + +- `auto_register_schemas` keyword argument has been added to `AvroSerializer`, which will allow for automatically registering schemas passed in to the `serialize`, when set to `True`, otherwise `False` by default. +- `value` parameter in `serialize` on `AvroSerializer` takes type `Mapping` rather than `Dict`. +- Depends on `azure-schemaregistry==1.0.0b3`. + +### Breaking Changes + +- `SchemaRegistryAvroSerializer` has been renamed `AvroSerializer`. +- `schema_registry` parameter in the `AvroSerializer` constructor has been renamed `client`. +- `schema_group` parameter in the `AvroSerializer` constructor has been renamed `group_name`. +- `data` parameter in the `serialize` and `deserialize` methods on `AvroSerializer` has been renamed `value`. +- `schema` parameter in the `serialize` method on `AvroSerializer` no longer accepts argument of type `bytes`. +- `AvroSerializer` constructor no longer takes in the `codec` keyword argument. +- The following positional arguments are now required keyword arguments: + - `client` and `group_name` in `AvroSerializer` constructor + - `schema` in `serialize` on `AvroSerializer` + +## 1.0.0b2 (2021-08-18) + +This version and all future versions will require Python 2.7 or Python 3.6+, Python 3.5 is no longer supported. + +### Features Added + +- Depends on `azure-schemaregistry==1.0.0b2` which supports client-level caching. + +## 1.0.0b1 (2020-09-09) + +Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry Avro Serializer. + +**New features** + +- `SchemaRegistryAvroSerializer` is the top-level client class that provides the functionality to encode and decode avro data utilizing the avro library. It will automatically register schema and retrieve schema from Azure Schema Registry Service. It provides two methods: + - `serialize`: Serialize dict data into bytes according to the given schema and register schema if needed. + - `deserialize`: Deserialize bytes data into dict data by automatically retrieving schema from the service. diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/LICENSE b/sdk/schemaregistry/azure-schemaregistry-avroserializer/LICENSE new file mode 100644 index 000000000000..63447fd8bbbf --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) Microsoft Corporation. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in b/sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in new file mode 100644 index 000000000000..a45dacf0a3f8 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/MANIFEST.in @@ -0,0 +1,6 @@ +include *.md +include LICENSE +include azure/__init__.py +include azure/schemaregistry/__init__.py +recursive-include tests *.py +recursive-include samples *.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md new file mode 100644 index 000000000000..e3d298467157 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md @@ -0,0 +1,329 @@ +# Azure Schema Registry Avro Serializer client library for Python + +Azure Schema Registry is a schema repository service hosted by Azure Event Hubs, providing schema storage, versioning, +and management. This package provides an Avro serializer capable of serializing and deserializing payloads containing +Schema Registry schema identifiers and Avro-serialized data. + +[Source code][source_code] | [Package (PyPi)][pypi] | [API reference documentation][api_reference] | [Samples][sr_avro_samples] | [Changelog][change_log] + +## _Disclaimer_ + +_Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to https://github.com/Azure/azure-sdk-for-python/issues/20691_ + +## Getting started + +### Install the package + +Install the Azure Schema Registry Avro Serializer client library and Azure Identity client library for Python with [pip][pip]: + +```Bash +pip install azure-schemaregistry-avroserializer azure-identity +``` + +### Prerequisites: +To use this package, you must have: +* Azure subscription - [Create a free account][azure_sub] +* [Azure Schema Registry][schemaregistry_service] +* Python 3.6 or later - [Install Python][python] + +### Authenticate the client +Interaction with the Schema Registry Avro Serializer starts with an instance of AvroSerializer class, which takes the schema group name and the [Schema Registry Client][schemaregistry_client] class. The client constructor takes the Event Hubs fully qualified namespace and and Azure Active Directory credential: + +* The fully qualified namespace of the Schema Registry instance should follow the format: `.servicebus.windows.net`. + +* An AAD credential that implements the [TokenCredential][token_credential_interface] protocol should be passed to the constructor. There are implementations of the `TokenCredential` protocol available in the +[azure-identity package][pypi_azure_identity]. To use the credential types provided by `azure-identity`, please install the Azure Identity client library for Python with [pip][pip]: + +```Bash +pip install azure-identity +``` + +* Additionally, to use the async API, you must first install an async transport, such as [aiohttp](https://pypi.org/project/aiohttp/): + +```Bash +pip install aiohttp +``` + +**Create AvroSerializer using the azure-schemaregistry library:** + +```python +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.identity import DefaultAzureCredential + +credential = DefaultAzureCredential() +# Namespace should be similar to: '.servicebus.windows.net' +fully_qualified_namespace = '<< FULLY QUALIFIED NAMESPACE OF THE SCHEMA REGISTRY >>' +group_name = '<< GROUP NAME OF THE SCHEMA >>' +schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, credential) +serializer = AvroSerializer(client=schema_registry_client, group_name=group_name) +``` + +## Key concepts + +### AvroSerializer + +Provides API to serialize to and deserialize from Avro Binary Encoding plus a +content type with schema ID. Uses [SchemaRegistryClient][schemaregistry_client] to get schema IDs from schema content or vice versa. + +### Supported message models + +Support has been added to certain Azure Messaging SDK model classes for interoperability with the `AvroSerializer`. These models are subtypes of the `MessageType` protocol defined under the `azure.schemaregistry.serializer.avroserializer` namespace. Currently, the supported model classes are: + +- `azure.eventhub.EventData` for `azure-eventhub==5.9.0b1` + +### Message format + +If a message type that follows the MessageType protocol is provided to the serializer, it will serialize the corresponding data and content type properties as follows: + +- `data`: Avro payload (in general, format-specific payload) + - Avro Binary Encoding + - NOT Avro Object Container File, which includes the schema and defeats the + purpose of this serializer to move the schema out of the message payload and + into the schema registry. + +- `content type`: a string of the format `avro/binary+`, where: + - `avro/binary` is the format indicator + - `` is the hexadecimal representation of GUID, same format and byte order as the string from the Schema Registry service. + +If message type or callback function is not provided, and by default, the serializer will create the following dict: +`{"data": , "content_type": 'avro/binary+' }` + +## Examples + +The following sections provide several code snippets covering some of the most common Schema Registry tasks, including: + +- [Serializing](#serializing) +- [Deserializing](#deserializing) +- [Event Hubs Sending Integration](#event-hubs-sending-integration) +- [Event Hubs Receiving Integration](#event-hubs-receiving-integration) + +### Serializing + +Use `AvroSerializer.serialize` method to serialize dict data with the given Avro schema. +The method will use a schema previously registered to the Schema Registry service and keep the schema cached for future serializing usage. It is also possible to avoid pre-registering the schema to the service and automatically register with the `serialize` method by instantiating the `AvroSerializer` with the keyword argument `auto_register_schemas=True`. + +```python +import os +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.identity import DefaultAzureCredential +from azure.eventhub import EventData + +token_credential = DefaultAzureCredential() +fully_qualified_namespace = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +group_name = "" +name = "example.avro.User" +format = "Avro" + +definition = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + +schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, token_credential) +schema_register_client.register(group_name, name, definition, format) +serializer = AvroSerializer(client=schema_registry_client, group_name=group_name) + +with serializer: + dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + event_data = serializer.serialize(dict_data, schema=definition, message_type=EventData) + + # OR + + metadata_dict = serializer.serialize(dict_data, schema=definition) + event_data = EventData.from_message_data(metadata_dict["data"], metadata_dict["content_type"]) +``` + +### Deserializing + +Use `AvroSerializer.deserialize` method to deserialize the bytes value into dict data by either: + - Passing in a message object that is a subtype of the `MessageType` protocol. + - Passing in a dict with keys `data`(type bytes) and `content_type` (type string). +The method automatically retrieves the schema from the Schema Registry Service and keeps the schema cached for future deserializing usage. + +```python +import os +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.identity import DefaultAzureCredential + +token_credential = DefaultAzureCredential() +fully_qualified_namespace = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +group_name = "" + +schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, token_credential) +serializer = AvroSerializer(client=schema_registry_client, group_name=group_name) + +with serializer: + # event_data is an EventData object with Avro serialized body + deserialized_data = serializer.deserialize(event_data) + + # OR + + serialized_bytes = b'' + content_type = 'avro/binary+' + data_dict = {"data": serialized_bytes, "content_type": content_type} + deserialized_data = serializer.deserialize(data_dict) +``` + +### Event Hubs Sending Integration + +Integration with [Event Hubs][eventhubs_repo] to send serialized Avro dict data as the body of EventData. + +```python +import os +from azure.eventhub import EventHubProducerClient, EventData +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.identity import DefaultAzureCredential + +token_credential = DefaultAzureCredential() +fully_qualified_namespace = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +group_name = "" +eventhub_connection_str = os.environ['EVENT_HUB_CONN_STR'] +eventhub_name = os.environ['EVENT_HUB_NAME'] + +definition = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + +schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, token_credential) +avro_serializer = AvroSerializer(client=schema_registry_client, group_name=group_name, auto_register_schemas=True) + +eventhub_producer = EventHubProducerClient.from_connection_string( + conn_str=eventhub_connection_str, + eventhub_name=eventhub_name +) + +with eventhub_producer, avro_serializer: + event_data_batch = eventhub_producer.create_batch() + dict_data = {"name": "Bob", "favorite_number": 7, "favorite_color": "red"} + event_data = avro_serializer.serialize(dict_data, schema=definition, message_type=EventData) + event_data_batch.add(event_data) + eventhub_producer.send_batch(event_data_batch) +``` + +### Event Hubs Receiving Integration + +Integration with [Event Hubs][eventhubs_repo] to receive `EventData` and deserialized raw bytes into Avro dict data. + +```python +import os +from azure.eventhub import EventHubConsumerClient +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.identity import DefaultAzureCredential + +token_credential = DefaultAzureCredential() +fully_qualified_namespace = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +group_name = "" +eventhub_connection_str = os.environ['EVENT_HUB_CONN_STR'] +eventhub_name = os.environ['EVENT_HUB_NAME'] + +schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, token_credential) +avro_serializer = AvroSerializer(client=schema_registry_client, group_name=group_name) + +eventhub_consumer = EventHubConsumerClient.from_connection_string( + conn_str=eventhub_connection_str, + consumer_group='$Default', + eventhub_name=eventhub_name, +) + +def on_event(partition_context, event): + deserialized_data = avro_serializer.deserialize(event) + +with eventhub_consumer, avro_serializer: + eventhub_consumer.receive(on_event=on_event, starting_position="-1") +``` + +## Troubleshooting + +### General + +Azure Schema Registry Avro Serializer raises exceptions defined in [Azure Core][azure_core]. + +### Logging +This library uses the standard +[logging][python_logging] library for logging. +Basic information about HTTP sessions (URLs, headers, etc.) is logged at INFO +level. + +Detailed DEBUG level logging, including request/response bodies and unredacted +headers, can be enabled on a client with the `logging_enable` argument: +```python +import sys +import logging +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.identity import DefaultAzureCredential + +# Create a logger for the SDK +logger = logging.getLogger('azure.schemaregistry') +logger.setLevel(logging.DEBUG) + +# Configure a console output +handler = logging.StreamHandler(stream=sys.stdout) +logger.addHandler(handler) + +credential = DefaultAzureCredential() +schema_registry_client = SchemaRegistryClient("", credential, logging_enable=True) +# This client will log detailed information about its HTTP sessions, at DEBUG level +serializer = AvroSerializer(client=schema_registry_client, group_name="") +``` + +Similarly, `logging_enable` can enable detailed logging for a single operation, +even when it isn't enabled for the client: +```py +serializer.serialize(dict_data, schema=schema_definition, logging_enable=True) +``` + +## Next steps + +### More sample code + +Please find further examples in the [samples][sr_avro_samples] directory demonstrating common Azure Schema Registry Avro Serializer scenarios. + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opserialize@microsoft.com](mailto:opserialize@microsoft.com) with any additional questions or comments. + + +[pip]: https://pypi.org/project/pip/ +[pypi]: https://pypi.org/project/azure-schemaregistry-avroserializer/ +[python]: https://www.python.org/downloads/ +[azure_core]: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/README.md +[azure_sub]: https://azure.microsoft.com/free/ +[python_logging]: https://docs.python.org/3/library/logging.html +[sr_avro_samples]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples +[api_reference]: https://docs.microsoft.com/python/api/overview/azure/schemaregistry-avroserializer-readme +[source_code]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer +[change_log]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md +[schemaregistry_client]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry +[schemaregistry_service]: https://aka.ms/schemaregistry +[eventhubs_repo]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/eventhub/azure-eventhub +[token_credential_interface]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/core/azure-core/azure/core/credentials.py +[pypi_azure_identity]: https://pypi.org/project/azure-identity/ \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/__init__.py new file mode 100644 index 000000000000..80f86cb969ec --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/__init__.py @@ -0,0 +1,26 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py new file mode 100644 index 000000000000..80f86cb969ec --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py @@ -0,0 +1,26 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py new file mode 100644 index 000000000000..80f86cb969ec --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py @@ -0,0 +1,26 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/__init__.py new file mode 100644 index 000000000000..4416b0bbcbcf --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/__init__.py @@ -0,0 +1,37 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from ._version import VERSION + +__version__ = VERSION + +from ._schema_registry_avro_serializer import AvroSerializer +from ._message_protocol import MessageType, MessageMetadataDict + +__all__ = [ + "AvroSerializer", + "MessageType", + "MessageMetadataDict" +] diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_abstract_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_abstract_avro_serializer.py new file mode 100644 index 000000000000..dc6711b336a3 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_abstract_avro_serializer.py @@ -0,0 +1,68 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from typing import BinaryIO, TypeVar, Union, Optional +from abc import abstractmethod + +ObjectType = TypeVar("ObjectType") + +class AbstractAvroObjectSerializer(object): + """ + An Avro serializer used for serializing/deserializing an Avro RecordSchema. + """ + + @abstractmethod + def get_schema_fullname( + self, + schema, + ): + # type: (str) -> str + """ + Returns the namespace-qualified name of the provided schema. + Schema must be a Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + :param schema: An Avro RecordSchema + :type schema: str + :rtype: str + """ + + + @abstractmethod + def serialize( + self, + data, + schema, + ): + # type: (ObjectType, str) -> bytes + """Convert the provided value to it's binary representation and write it to the stream. + Schema must be a Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + :param data: An object to serialize + :type data: ObjectType + :param schema: An Avro RecordSchema + :type schema: str + :returns: Serialized bytes + :rtype: bytes + """ + + @abstractmethod + def deserialize( + self, + data: Union[bytes, BinaryIO], + schema: str, + *, + readers_schema: Optional[str] + ): + """Read the binary representation into a specific type. + Return type will be ignored, since the schema is deduced from the provided bytes. + :param data: A stream of bytes or bytes directly + :type data: BinaryIO or bytes + :param schema: An Avro RecordSchema + :type schema: str + :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. + :paramtype readers_schema: str or None + :returns: An instantiated object + :rtype: ObjectType + """ diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_apache_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_apache_avro_serializer.py new file mode 100644 index 000000000000..a1295800ebbc --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_apache_avro_serializer.py @@ -0,0 +1,100 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from functools import lru_cache +from typing import BinaryIO, Union, TypeVar +from io import BytesIO +import avro +from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder + +from ._abstract_avro_serializer import AbstractAvroObjectSerializer + +ObjectType = TypeVar("ObjectType") + + +class ApacheAvroObjectSerializer(AbstractAvroObjectSerializer): + + def __init__(self, codec=None): + """A Avro serializer using avro lib from Apache. + :param str codec: The writer codec. If None, let the avro library decides. + """ + self._writer_codec = codec + + @lru_cache(maxsize=128) + def parse_schema(self, schema): # pylint: disable=no-self-use + return avro.schema.parse(schema) + + def get_schema_fullname(self, schema): + parsed_schema = self.parse_schema(schema) + return parsed_schema.fullname + + @lru_cache(maxsize=128) + def get_schema_writer(self, schema): # pylint: disable=no-self-use + schema = self.parse_schema(schema) + return DatumWriter(schema) + + @lru_cache(maxsize=128) + def get_schema_reader(self, schema, readers_schema): # pylint: disable=no-self-use + schema = self.parse_schema(schema) + if readers_schema: + readers_schema = self.parse_schema(readers_schema) + return DatumReader(writers_schema=schema, readers_schema=readers_schema) + + # pylint: disable=no-self-use + def serialize( + self, + data, # type: ObjectType + schema, # type: Union[str, bytes, avro.schema.Schema] + ) -> bytes: + """Convert the provided value to it's binary representation and write it to the stream. + Schema must be a Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + :param data: An object to serialize + :type data: ObjectType + :param schema: An Avro RecordSchema + :type schema: str + :returns: Serialized bytes + :rtype: bytes + """ + if not schema: + raise ValueError("Schema is required in Avro serializer.") + + writer = self.get_schema_writer(schema) + + stream = BytesIO() + with stream: + writer.write(data, BinaryEncoder(stream)) + serialized_data = stream.getvalue() + return serialized_data + + # pylint: disable=no-self-use + def deserialize( + self, + data, # type: Union[bytes, BinaryIO] + schema, # type: Union[str, bytes, avro.schema.Schema] + *, + readers_schema=None, # type: Optional[Union[str, bytes, avro.schema.Schema]] + ) -> ObjectType: + """Read the binary representation into a specific type. + Return type will be ignored, since the schema is deduced from the provided bytes. + :param data: A stream of bytes or bytes directly + :type data: BinaryIO or bytes + :param schema: An Avro RecordSchema + :type schema: str + :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. + :paramtype readers_schema: str or None + :returns: An instantiated object + :rtype: ObjectType + """ + if not hasattr(data, 'read'): + data = BytesIO(data) + + reader = self.get_schema_reader(schema, readers_schema) + + with data: + bin_decoder = BinaryDecoder(data) + deserialized_data = reader.read(bin_decoder) + + return deserialized_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_constants.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_constants.py new file mode 100644 index 000000000000..2deb93932e05 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_constants.py @@ -0,0 +1,40 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +AVRO_MIME_TYPE = "avro/binary" + +# encoded payload used to consist of: +# 0~3: 4 bytes denoting record format identifier +# 4~35: 32 bytes denoting schema id +# 36~END: encode data + +# keeping temporarily for backward compatibility + +RECORD_FORMAT_IDENTIFIER_START_INDEX = 0 +RECORD_FORMAT_IDENTIFIER_LENGTH = 4 +SCHEMA_ID_START_INDEX = 4 +SCHEMA_ID_LENGTH = 32 +DATA_START_INDEX = 36 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_message_protocol.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_message_protocol.py new file mode 100644 index 000000000000..ee69f0c10fe7 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_message_protocol.py @@ -0,0 +1,39 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any +try: + from typing import Protocol, TypedDict +except ImportError: + from typing_extensions import Protocol, TypedDict + + +class MessageMetadataDict(TypedDict): + """A dict with required keys: + - `data`: bytes + - `content_type`: str + """ + + data: bytes + content_type: str + +class MessageType(Protocol): + """Message Types that set and get data and content type values internally. + """ + + @classmethod + def from_message_data(cls, data: bytes, content_type: str, **kwargs: Any) -> "MessageType": + """ + Creates an object that is a subtype of MessageType given content type and + a data value to be set as body. + + :param bytes data: The data value to be set as the body of the message. + :param str content_type: The content type to be set on the message. + :rtype: ~azure.schemaregistry.encoder.avroencoder.MessageType + """ + ... + + def __message_data__(self) -> MessageMetadataDict: + ... diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py new file mode 100644 index 000000000000..afdb959042e0 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -0,0 +1,285 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from functools import lru_cache +from io import BytesIO +from typing import Any, Dict, Mapping, Optional, Union, Callable + +from .exceptions import ( + SchemaParseError, + SchemaSerializationError, + SchemaDeserializationError, +) +from ._apache_avro_serializer import ApacheAvroObjectSerializer as AvroObjectSerializer +from ._message_protocol import MessageMetadataDict, MessageType +from ._constants import ( + SCHEMA_ID_START_INDEX, + SCHEMA_ID_LENGTH, + DATA_START_INDEX, + AVRO_MIME_TYPE, + RECORD_FORMAT_IDENTIFIER_LENGTH, +) + + +class AvroSerializer(object): + """ + AvroSerializer provides the ability to serialize and deserialize data according + to the given avro schema. It would automatically register, get and cache the schema. + + :keyword client: Required. The schema registry client + which is used to register schema and retrieve schema from the service. + :paramtype client: ~azure.schemaregistry.SchemaRegistryClient + :keyword str group_name: Required. Schema group under which schema should be registered. + :keyword bool auto_register_schemas: When true, register new schemas passed to serialize. + Otherwise, and by default, serialize will fail if the schema has not been pre-registered in the registry. + + """ + + def __init__(self, **kwargs): + # type: (Any) -> None + try: + self._schema_group = kwargs.pop("group_name") + self._schema_registry_client = kwargs.pop( + "client" + ) # type: "SchemaRegistryClient" + except KeyError as e: + raise TypeError("'{}' is a required keyword.".format(e.args[0])) + self._avro_serializer = AvroObjectSerializer(codec=kwargs.get("codec")) + self._auto_register_schemas = kwargs.get("auto_register_schemas", False) + self._auto_register_schema_func = ( + self._schema_registry_client.register_schema + if self._auto_register_schemas + else self._schema_registry_client.get_schema_properties + ) + + def __enter__(self): + # type: () -> AvroSerializer + self._schema_registry_client.__enter__() + return self + + def __exit__(self, *exc_details): + # type: (Any) -> None + self._schema_registry_client.__exit__(*exc_details) + + def close(self): + # type: () -> None + """This method is to close the sockets opened by the client. + It need not be used when using with a context manager. + """ + self._schema_registry_client.close() + + @lru_cache(maxsize=128) + def _get_schema_id(self, schema_name, schema_str, **kwargs): + # type: (str, str, Any) -> str + """ + Get schema id from local cache with the given schema. + If there is no item in the local cache, get schema id from the service and cache it. + + :param schema_name: Name of the schema + :type schema_name: str + :param str schema_str: Schema string + :return: Schema Id + :rtype: str + """ + schema_id = self._auto_register_schema_func( + self._schema_group, schema_name, schema_str, "Avro", **kwargs + ).id + return schema_id + + @lru_cache(maxsize=128) + def _get_schema(self, schema_id, **kwargs): + # type: (str, Any) -> str + """ + Get schema content from local cache with the given schema id. + If there is no item in the local cache, get schema from the service and cache it. + + :param str schema_id: Schema id + :return: Schema content + :rtype: str + """ + schema_str = self._schema_registry_client.get_schema( + schema_id, **kwargs + ).definition + return schema_str + + def serialize( + self, + data: Mapping[str, Any], + *, + schema: str, + message_type: Optional[Callable] = None, + **kwargs: Any, + ) -> Union[MessageType, MessageMetadataDict]: + """ + Serialize data with the given schema. Create content type value, which consists of the Avro Mime Type string + and the schema ID corresponding to given schema. If provided with a message constructor callback, + pass serialized data and content type to create message object. If not provided, return the following dict: + {"data": Avro serialized value, "content_type": Avro mime type string + schema ID}. + + If `message_type` is set, then additional keyword arguments will be passed to the message callback + function provided. + + Schema must be an Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + + :param data: The data to be serialized. + :type data: Mapping[str, Any] + :keyword schema: Required. The schema used to serialize the data. + :paramtype schema: str + :keyword message_type: The callback function or message class to construct the message. If message class, + it must be a subtype of the azure.schemaregistry.serializer.avroserializer.MessageType protocol. + If callback function, it must have the following method signature: + `(data: bytes, content_type: str, **kwargs) -> MessageType`, where `data` and `content_type` + are positional parameters. + :paramtype message_type: Callable or None + :rtype: MessageType or MessageMetadataDict + :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaParseError: + Indicates an issue with parsing schema. + :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaSerializationError: + Indicates an issue with serializing data for provided schema. + """ + + raw_input_schema = schema + + try: + schema_fullname = self._avro_serializer.get_schema_fullname(raw_input_schema) + except Exception as e: # pylint:disable=broad-except + SchemaParseError( + f"Cannot parse schema: {raw_input_schema}", error=e + ).raise_with_traceback() + + schema_id = self._get_schema_id(schema_fullname, raw_input_schema) + content_type = f"{AVRO_MIME_TYPE}+{schema_id}" + + try: + data_bytes = self._avro_serializer.serialize(data, raw_input_schema) + except Exception as e: # pylint:disable=broad-except + SchemaSerializationError( + "Cannot serialize value '{}' for schema: {}".format( + data, raw_input_schema + ), + error=e, + ).raise_with_traceback() + + stream = BytesIO() + + stream.write(data_bytes) + stream.flush() + + payload = stream.getvalue() + stream.close() + + if message_type: + try: + return message_type.from_message_data(payload, content_type, **kwargs) + except AttributeError: + try: + return message_type(payload, content_type, **kwargs) + except TypeError as e: + SchemaSerializationError( + f"""The data model {str(message_type)} is not a Callable that takes `data` + and `content_type` or a subtype of the MessageType protocol. + If using an Azure SDK model class, please check the README.md for the full list + of supported Azure SDK models and their corresponding versions.""" + ).raise_with_traceback() + + return {"data": payload, "content_type": content_type} + + def _convert_preamble_format( + self, data, content_type + ): # pylint: disable=no-self-use + record_format_identifier = b"\0\0\0\0" + if data[0:RECORD_FORMAT_IDENTIFIER_LENGTH] == record_format_identifier: + schema_id = data[ + SCHEMA_ID_START_INDEX : (SCHEMA_ID_START_INDEX + SCHEMA_ID_LENGTH) + ].decode("utf-8") + content_type = f"{AVRO_MIME_TYPE}+{schema_id}" + data = data[DATA_START_INDEX:] + + return data, content_type + + def deserialize( + self, + message: Union[MessageType, MessageMetadataDict], + *, + readers_schema: Optional[str] = None, + **kwargs, # pylint: disable=unused-argument + ) -> Dict[str, Any]: + """ + Deserialize bytes data using schema ID in the content type field. `message` must be one of the following: + 1) A Subtype of the MessageType protocol. + 2) A dict {"data": ..., "content_type": ...}, where "data" is bytes and "content_type" is string. + 3) If using to deserialize data that was serialized with the AvroSerializer, a dict + {"data": ..., "content_type": None}, where "data" is bytes and "content_type" is None. + Data must follow format of associated Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + + :param message: The message object which holds the data to be deserialized and content type + containing the schema ID. + :type message: MessageType or MessageMetadataDict + :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. + :paramtype readers_schema: str or None + :rtype: Dict[str, Any] + :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaParseError: + Indicates an issue with parsing schema. + :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaDeserializationError: + Indicates an issue with decoding value. + """ + + try: + message_data_dict = message.__message_data__() + data = message_data_dict["data"] + content_type = message_data_dict["content_type"] + except AttributeError: + try: + data = message["data"] + content_type = message["content_type"] + except (KeyError, TypeError): + SchemaDeserializationError( + f"""The data model {str(message)} is not a subtype of the MessageType protocol or type + MessageMetadataDict. If using an Azure SDK model class, please check the README.md + for the full list of supported Azure SDK models and their corresponding versions.""" + ).raise_with_traceback() + + # include in first preview for back compatibility + data, content_type = self._convert_preamble_format(data, content_type) + + schema_id = content_type.split("+")[1] + schema_definition = self._get_schema(schema_id) + try: + dict_value = self._avro_serializer.deserialize( + data, schema_definition, readers_schema=readers_schema + ) + except Exception as e: # pylint:disable=broad-except + error_message = ( + f"Cannot deserialize value '{data}' for schema: {schema_definition}\n and reader's schema: {readers_schema}" + if readers_schema + else f"Cannot deserialize value '{data}' for schema: {schema_definition}" + ) + SchemaDeserializationError( + error_message, + error=e, + ).raise_with_traceback() + return dict_value diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_version.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_version.py new file mode 100644 index 000000000000..e7eb08280bf8 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_version.py @@ -0,0 +1,27 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +VERSION = "1.0.0b5" diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/__init__.py new file mode 100644 index 000000000000..14d743e38828 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/__init__.py @@ -0,0 +1,30 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from ._schema_registry_avro_serializer_async import AvroSerializer + +__all__ = [ + "AvroSerializer" +] diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_async_lru.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_async_lru.py new file mode 100644 index 000000000000..c51dfdf0c9dd --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_async_lru.py @@ -0,0 +1,230 @@ +# -------------------------------------------------------------------------- +# The MIT License +# +# Copyright (c) 2018 aio-libs team https://github.com/aio-libs/ +# Copyright (c) 2017 Ocean S. A. https://ocean.io/ +# Copyright (c) 2016-2017 WikiBusiness Corporation http://wikibusiness.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# -------------------------------------------------------------------------- +# Copying over `async_lru.py`[https://github.com/aio-libs/async-lru/blob/master/async_lru.py] +# from `aio-libs`[https://github.com/aio-libs/async-lru] for the following reasons: +# 1. There has not been an official release of `async_lru` in 2 years. +# 2. The last update to the library was a year ago, so it seems the library is +# not being actively maintained. + +import asyncio +from collections import OrderedDict +from functools import _CacheInfo, _make_key, partial, wraps + + +__version__ = "1.0.2" + +__all__ = ("alru_cache",) + + +def unpartial(fn): + while hasattr(fn, "func"): + fn = fn.func + + return fn + + +def _done_callback(fut, task): + if task.cancelled(): + fut.cancel() + return + + exc = task.exception() + if exc is not None: + fut.set_exception(exc) + return + + fut.set_result(task.result()) + + +def _cache_invalidate(wrapped, typed, *args, **kwargs): + # pylint: disable=protected-access + key = _make_key(args, kwargs, typed) + + exists = key in wrapped._cache + + if exists: + wrapped._cache.pop(key) + + return exists + + +def _cache_clear(wrapped): + # pylint: disable=protected-access + wrapped.hits = wrapped.misses = 0 + wrapped._cache = OrderedDict() + wrapped.tasks = set() + + +def _open(wrapped): + if not wrapped.closed: + raise RuntimeError("alru_cache is not closed") + + # pylint: disable=protected-access + was_closed = ( + wrapped.hits == wrapped.misses == len(wrapped.tasks) == len(wrapped._cache) == 0 + ) + + if not was_closed: + raise RuntimeError("alru_cache was not closed correctly") + + wrapped.closed = False + + +def _close(wrapped, *, cancel=False, return_exceptions=True): + if wrapped.closed: + raise RuntimeError("alru_cache is closed") + + wrapped.closed = True + + if cancel: + for task in wrapped.tasks: + if not task.done(): # not sure is it possible + task.cancel() + + return _wait_closed(wrapped, return_exceptions=return_exceptions) + + +async def _wait_closed(wrapped, *, return_exceptions): + wait_closed = asyncio.gather(*wrapped.tasks, return_exceptions=return_exceptions) + + wait_closed.add_done_callback(partial(_close_waited, wrapped)) + + ret = await wait_closed + + # hack to get _close_waited callback to be executed + await asyncio.sleep(0) + + return ret + + +def _close_waited(wrapped, _): + wrapped.cache_clear() + + +def _cache_info(wrapped, maxsize): + # pylint: disable=protected-access + return _CacheInfo( + wrapped.hits, + wrapped.misses, + maxsize, + len(wrapped._cache), + ) + + +def __cache_touch(wrapped, key): + # pylint: disable=protected-access + try: + wrapped._cache.move_to_end(key) + except KeyError: # not sure is it possible + pass + + +def _cache_hit(wrapped, key): + wrapped.hits += 1 + __cache_touch(wrapped, key) + + +def _cache_miss(wrapped, key): + wrapped.misses += 1 + __cache_touch(wrapped, key) + + +def alru_cache( + fn=None, + maxsize=128, + typed=False, + *, + cache_exceptions=True, +): + def wrapper(fn): + # pylint: disable=protected-access + _origin = unpartial(fn) + + if not asyncio.iscoroutinefunction(_origin): + raise RuntimeError("Coroutine function is required, got {}".format(fn)) + + # functools.partialmethod support + if hasattr(fn, "_make_unbound_method"): + fn = fn._make_unbound_method() + + @wraps(fn) + async def wrapped(*fn_args, **fn_kwargs): + if wrapped.closed: + raise RuntimeError("alru_cache is closed for {}".format(wrapped)) + + loop = asyncio.get_event_loop() + + key = _make_key(fn_args, fn_kwargs, typed) + + fut = wrapped._cache.get(key) + + if fut is not None: + if not fut.done(): + _cache_hit(wrapped, key) + return await asyncio.shield(fut) + + exc = fut._exception + + if exc is None or cache_exceptions: + _cache_hit(wrapped, key) + return fut.result() + + # exception here and cache_exceptions == False + wrapped._cache.pop(key) + + fut = loop.create_future() + task = loop.create_task(fn(*fn_args, **fn_kwargs)) + task.add_done_callback(partial(_done_callback, fut)) + + wrapped.tasks.add(task) + task.add_done_callback(wrapped.tasks.remove) + + wrapped._cache[key] = fut + + if maxsize is not None and len(wrapped._cache) > maxsize: + wrapped._cache.popitem(last=False) + + _cache_miss(wrapped, key) + return await asyncio.shield(fut) + + _cache_clear(wrapped) + wrapped._origin = _origin + wrapped.closed = False + wrapped.cache_info = partial(_cache_info, wrapped, maxsize) + wrapped.cache_clear = partial(_cache_clear, wrapped) + wrapped.invalidate = partial(_cache_invalidate, wrapped, typed) + wrapped.close = partial(_close, wrapped) + wrapped.open = partial(_open, wrapped) + + return wrapped + + if fn is None: + return wrapper + + if callable(fn) or hasattr(fn, "_make_unbound_method"): + return wrapper(fn) + + raise NotImplementedError("{} decorating is not supported".format(fn)) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py new file mode 100644 index 000000000000..ce472a54a717 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py @@ -0,0 +1,277 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from io import BytesIO +from typing import Any, Callable, Dict, Mapping, Union, Optional +from ._async_lru import alru_cache +from .._constants import ( + SCHEMA_ID_START_INDEX, + SCHEMA_ID_LENGTH, + DATA_START_INDEX, + AVRO_MIME_TYPE, + RECORD_FORMAT_IDENTIFIER_LENGTH, +) +from .._message_protocol import MessageType, MessageMetadataDict +from .._apache_avro_serializer import ApacheAvroObjectSerializer as AvroObjectSerializer +from ..exceptions import ( + SchemaParseError, + SchemaSerializationError, + SchemaDeserializationError, +) + + +class AvroSerializer(object): + """ + AvroSerializer provides the ability to serialize and deserialize data according + to the given avro schema. It would automatically register, get and cache the schema. + + :keyword client: Required. The schema registry client + which is used to register schema and retrieve schema from the service. + :paramtype client: ~azure.schemaregistry.aio.SchemaRegistryClient + :keyword str group_name: Required. Schema group under which schema should be registered. + :keyword bool auto_register_schemas: When true, register new schemas passed to serialize. + Otherwise, and by default, serialize will fail if the schema has not been pre-registered in the registry. + + """ + + def __init__(self, **kwargs): + # type: (Any) -> None + try: + self._schema_group = kwargs.pop("group_name") + self._schema_registry_client = kwargs.pop( + "client" + ) # type: "SchemaRegistryClient" + except KeyError as e: + raise TypeError("'{}' is a required keyword.".format(e.args[0])) + self._avro_serializer = AvroObjectSerializer(codec=kwargs.get("codec")) + self._auto_register_schemas = kwargs.get("auto_register_schemas", False) + self._auto_register_schema_func = ( + self._schema_registry_client.register_schema + if self._auto_register_schemas + else self._schema_registry_client.get_schema_properties + ) + + async def __aenter__(self): + # type: () -> AvroSerializer + await self._schema_registry_client.__aenter__() + return self + + async def __aexit__(self, *exc_details): + # type: (Any) -> None + await self._schema_registry_client.__aexit__(*exc_details) + + async def close(self): + # type: () -> None + """This method is to close the sockets opened by the client. + It need not be used when using with a context manager. + """ + await self._schema_registry_client.close() + + @alru_cache(maxsize=128, cache_exceptions=False) + async def _get_schema_id(self, schema_name, schema_str, **kwargs): + # type: (str, str, Any) -> str + """ + Get schema id from local cache with the given schema. + If there is no item in the local cache, get schema id from the service and cache it. + + :param schema_name: Name of the schema + :type schema_name: str + :param str schema_str: Schema string + :return: Schema Id + :rtype: str + """ + schema_properties = await self._auto_register_schema_func( + self._schema_group, schema_name, schema_str, "Avro", **kwargs + ) + return schema_properties.id + + @alru_cache(maxsize=128, cache_exceptions=False) + async def _get_schema(self, schema_id, **kwargs): + # type: (str, Any) -> str + """ + Get schema definition from local cache with the given schema id. + If there is no item in the local cache, get schema from the service and cache it. + + :param str schema_id: Schema id + :return: Schema definition + """ + schema = await self._schema_registry_client.get_schema(schema_id, **kwargs) + return schema.definition + + async def serialize( + self, + data: Mapping[str, Any], + *, + schema: str, + message_type: Optional[Callable] = None, + **kwargs: Any, + ) -> Union[MessageType, MessageMetadataDict]: + + """ + Serialize data with the given schema. Create content type value, which consists of the Avro Mime Type string + and the schema ID corresponding to given schema. If provided with a message constructor callback, + pass serialized data and content type to create message object. If not provided, return the following dict: + {"data": Avro serialized value, "content_type": Avro mime type string + schema ID}. + + If `message_type` is set, then additional keyword arguments will be passed to the message callback + function provided. + + Schema must be an Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + + :param data: The data to be serialized. + :type data: Mapping[str, Any] + :keyword schema: Required. The schema used to serialize the data. + :paramtype schema: str + :keyword message_type: The callback function or message class to construct the message. If message class, + it must be a subtype of the azure.schemaregistry.serializer.avroserializer.MessageType protocol. + If callback function, it must have the following method signature: + `(data: bytes, content_type: str, **kwargs) -> MessageType`, where `data` and `content_type` + are positional parameters. + :paramtype message_type: Callable or None + :rtype: MessageType or MessageMetadataDict + :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaParseError: + Indicates an issue with parsing schema. + :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaSerializationError: + Indicates an issue with serializing data for provided schema. + """ + + raw_input_schema = schema + + try: + schema_fullname = self._avro_serializer.get_schema_fullname(raw_input_schema) + except Exception as e: # pylint:disable=broad-except + SchemaParseError( + f"Cannot parse schema: {raw_input_schema}", error=e + ).raise_with_traceback() + + schema_id = await self._get_schema_id(schema_fullname, raw_input_schema) + content_type = f"{AVRO_MIME_TYPE}+{schema_id}" + + try: + data_bytes = self._avro_serializer.serialize(data, raw_input_schema) + except Exception as e: # pylint:disable=broad-except + SchemaSerializationError( + "Cannot serialize value '{}' for schema: {}".format( + data, raw_input_schema + ), + error=e, + ).raise_with_traceback() + + stream = BytesIO() + + stream.write(data_bytes) + stream.flush() + + payload = stream.getvalue() + stream.close() + if message_type: + try: + return message_type.from_message_data(payload, content_type, **kwargs) + except AttributeError: + try: + return message_type(payload, content_type, **kwargs) + except TypeError as e: + SchemaSerializationError( + f"""The data model {str(message_type)} is not a Callable that takes `data` + and `content_type` or a subtype of the MessageType protocol. + If using an Azure SDK model class, please check the README.md for the full list + of supported Azure SDK models and their corresponding versions.""" + ).raise_with_traceback() + + return {"data": payload, "content_type": content_type} + + def _convert_preamble_format(self, data, content_type): # pylint: disable=no-self-use + record_format_identifier = b"\0\0\0\0" + if data[0:RECORD_FORMAT_IDENTIFIER_LENGTH] == record_format_identifier: + schema_id = data[ + SCHEMA_ID_START_INDEX : (SCHEMA_ID_START_INDEX + SCHEMA_ID_LENGTH) + ].decode("utf-8") + content_type = f"{AVRO_MIME_TYPE}+{schema_id}" + data = data[DATA_START_INDEX:] + + return data, content_type + + async def deserialize( + self, + message: Union[MessageType, MessageMetadataDict], + *, + readers_schema: Optional[str] = None, + **kwargs, # pylint: disable=unused-argument + ) -> Dict[str, Any]: + """ + Deserialize bytes data using schema ID in the content type field. `message` must be one of the following: + 1) A Subtype of the MessageType protocol. + 2) A dict {"data": ..., "content_type": ...}, where "data" is bytes and "content_type" is string. + 3) If using to deserialize data that was serialized with the AvroSerializer, a dict + {"data": ..., "content_type": None}, where "data" is bytes and "content_type" is None. + Data must follow format of associated Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema + + :param message: The message object which holds the data to be deserialized and content type + containing the schema ID. + :type message: MessageType or MessageMetadataDict + :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. + :paramtype readers_schema: str or None + :rtype: Dict[str, Any] + :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaParseError: + Indicates an issue with parsing schema. + :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaDeserializationError: + Indicates an issue with deserializing value. + """ + + try: + message_data_dict = message.__message_data__() + data = message_data_dict["data"] + content_type = message_data_dict["content_type"] + except AttributeError: + try: + data = message["data"] + content_type = message["content_type"] + except (KeyError, TypeError): + SchemaDeserializationError( + f"""The data model {str(message)} is not a subtype of the MessageType protocol or type + MessageMetadataDict. If using an Azure SDK model class, please check the README.md + for the full list of supported Azure SDK models and their corresponding versions.""" + ).raise_with_traceback() + + # include in first preview for back compatibility + data, content_type = self._convert_preamble_format(data, content_type) + + schema_id = content_type.split("+")[1] + schema_definition = await self._get_schema(schema_id) + try: + dict_value = self._avro_serializer.deserialize(data, schema_definition, readers_schema=readers_schema) + except Exception as e: # pylint:disable=broad-except + error_message = ( + f"Cannot deserialize value '{data}' for schema: {schema_definition}\n and reader's schema: {readers_schema}" + if readers_schema + else f"Cannot deserialize value '{data}' for schema: {schema_definition}" + ) + SchemaDeserializationError( + error_message, + error=e, + ).raise_with_traceback() + return dict_value diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/exceptions.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/exceptions.py new file mode 100644 index 000000000000..b8b19892b1cf --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/exceptions.py @@ -0,0 +1,69 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from azure.core.exceptions import AzureError + +class SchemaParseError(AzureError): + """Error parsing a JSON schema. + :param str message: The message object stringified as 'message' attribute + :keyword error: The original exception, if any + + :ivar str message: A stringified version of the message parameter + :ivar inner_exception: The exception passed with the 'error' kwarg + :vartype inner_exception: Exception + :ivar exc_type: The exc_type from sys.exc_info() + :ivar exc_value: The exc_value from sys.exc_info() + :ivar exc_traceback: The exc_traceback from sys.exc_info() + :ivar exc_msg: A string formatting of message parameter, exc_type and exc_value + """ + + +class SchemaSerializationError(AzureError): + """Error serializing a JSON schema. + :param str message: The message object stringified as 'message' attribute + :keyword error: The original exception, if any + + :ivar str message: A stringified version of the message parameter + :ivar inner_exception: The exception passed with the 'error' kwarg + :vartype inner_exception: Exception + :ivar exc_type: The exc_type from sys.exc_info() + :ivar exc_value: The exc_value from sys.exc_info() + :ivar exc_traceback: The exc_traceback from sys.exc_info() + :ivar exc_msg: A string formatting of message parameter, exc_type and exc_value + """ + +class SchemaDeserializationError(AzureError): + """Error deserializing a JSON schema. + :param str message: The message object stringified as 'message' attribute + :keyword error: The original exception, if any + + :ivar str message: A stringified version of the message parameter + :ivar inner_exception: The exception passed with the 'error' kwarg + :vartype inner_exception: Exception + :ivar exc_type: The exc_type from sys.exc_info() + :ivar exc_value: The exc_value from sys.exc_info() + :ivar exc_traceback: The exc_traceback from sys.exc_info() + :ivar exc_msg: A string formatting of message parameter, exc_type and exc_value + """ diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt new file mode 100644 index 000000000000..14549f3dbe61 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/dev_requirements.txt @@ -0,0 +1,4 @@ +-e ../../../tools/azure-devtools +-e ../../../tools/azure-sdk-tools +-e ../../identity/azure-identity +aiohttp>=3.0 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/mypy.ini b/sdk/schemaregistry/azure-schemaregistry-avroserializer/mypy.ini new file mode 100644 index 000000000000..c5adef3f8a6d --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +python_version = 3.7 +warn_unused_configs = True +ignore_missing_imports = True + +# Per-module options: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md new file mode 100644 index 000000000000..55cd06ca90f6 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/README.md @@ -0,0 +1,65 @@ +--- +page_type: sample +languages: + - python +products: + - azure + - azure-event-hubs +urlFragment: schemaregistry-avroserializer-samples +--- + +# Azure Schema Registry Avro serializer library for Python Samples + +These are code samples that show common scenario operations with the Schema Registry Avro serializer library. + +Several Schema Registry Avro serializer Python SDK samples are available to you in the SDK's GitHub repository. These samples provide example code for additional scenarios commonly encountered while working with Schema Registry Avro serializer: + +* [serialize_and_deserialize_event_data_message.py][serialize_and_deserialize_event_data_message_sample] ([async version][serialize_and_deserialize_event_data_message_async_sample]) - Examples for common Schema Registry Avro serializer tasks: + * Serialize data according to the given schema and create EventData object + * Deserialize data given an EventData object with serialized data and corresponding content type +* [eventhub_send_integration.py][eventhub_send_integration_sample] ([async version][eventhub_send_integration_async_sample]) - Examples for integration with EventHub in sending tasks: + * Serialize data with the given schema and send `EventData` to Event Hubs. +* [eventhub_receive_integration.py][eventhub_receive_integration_sample] ([async version][eventhub_receive_integration_async_sample]) - Examples for integration with EventHub in receiving tasks: + * Receive `EventData` from Event Hubs and deserialize the received bytes. + +## Prerequisites +- Python 3.6 or later. +- **Microsoft Azure Subscription:** To use Azure services, including Azure Schema Registry, you'll need a subscription. +If you do not have an existing Azure account, you may sign up for a free trial or use your MSDN subscriber benefits when you [create an account](https://account.windowsazure.com/Home/Index). + +## Setup + +1. Install the Azure Schema Registry Avro serializer client library and Azure Identity client library for Python with [pip](https://pypi.org/project/pip/): + +```bash +pip install azure-schemaregistry-avroserializer azure-identity +``` + +Additionally, if using with `azure.eventhub.EventData`, install `azure-eventhub==5.9.0b1`: + +```bash +pip install azure-eventhub==5.9.0b1 +``` + +2. Clone or download this sample repository +3. Open the sample folder in Visual Studio Code or your IDE of choice. + +## Running the samples + +1. Open a terminal window and `cd` to the directory that the samples are saved in. +2. Set the environment variables specified in the sample file you wish to run. +3. Follow the usage described in the file, e.g. `python serialize_and_deserialize_event_data_message.py` + +## Next steps + +Check out the [API reference documentation][api_reference] to learn more about +what you can do with the Azure Schema Registry Avro serializer library. + + +[serialize_and_deserialize_event_data_message_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_event_data_message.py +[eventhub_send_integration_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_send_integration.py +[eventhub_receive_integration_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_receive_integration.py +[serialize_and_deserialize_event_data_message_async_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_event_data_message_async.py +[eventhub_send_integration_async_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_send_integration_async.py +[eventhub_receive_integration_async_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_receive_integration_async.py +[api_reference]: https://docs.microsoft.com/python/api/overview/azure/schemaregistry-avroserializer-readme diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_receive_integration_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_receive_integration_async.py new file mode 100644 index 000000000000..b78970c00e12 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_receive_integration_async.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +Examples to show receiving events from EventHub with AvroSerializer integrated for data deserializing. +""" + +# pylint: disable=C0111 +import os +import asyncio +from azure.eventhub.aio import EventHubConsumerClient +from azure.identity.aio import DefaultAzureCredential +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer.aio import AvroSerializer + +EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] +EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] + + +# create an EventHubConsumerClient instance +eventhub_consumer = EventHubConsumerClient.from_connection_string( + conn_str=EVENTHUB_CONNECTION_STR, + consumer_group='$Default', + eventhub_name=EVENTHUB_NAME, +) +# create a AvroSerializer instance +azure_credential = DefaultAzureCredential() +avro_serializer = AvroSerializer( + client=SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=azure_credential + ), + group_name=GROUP_NAME, + auto_register_schemas=True +) + +async def on_event(partition_context, event): + print(f"Received event from partition: {partition_context.partition_id}.") + + bytes_payload = b"".join(b for b in event.body) + print(f'The received bytes of the EventData is {bytes_payload}.') + + # Use the deserialize method to deserialize the payload of the event. + # The deserialize method will extract the schema id from the content_type, and automatically retrieve the Avro Schema + # from the Schema Registry Service. The schema will be cached locally for future usage. + deserialized_data = await avro_serializer.deserialize(event) + print(f'The dict data after deserializing is {deserialized_data}') + + +async def main(): + try: + async with eventhub_consumer, avro_serializer: + await eventhub_consumer.receive( + on_event=on_event, + starting_position="-1", # "-1" is from the beginning of the partition. + ) + except KeyboardInterrupt: + print('Stopped receiving.') + finally: + await avro_serializer.close() + await azure_credential.close() + await eventhub_consumer.close() + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_send_integration_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_send_integration_async.py new file mode 100644 index 000000000000..b78d84a56ead --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/eventhub_send_integration_async.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +Examples to show sending event to EventHub with AvroSerializer integrated for data serialization. +""" + +# pylint: disable=C0111 + +import os +import asyncio +from azure.eventhub import EventData +from azure.eventhub.aio import EventHubProducerClient +from azure.identity.aio import DefaultAzureCredential +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer.aio import AvroSerializer + +EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] +EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] + +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + +# create an EventHubProducerClient instance +eventhub_producer = EventHubProducerClient.from_connection_string( + conn_str=EVENTHUB_CONNECTION_STR, + eventhub_name=EVENTHUB_NAME +) +# create a AvroSerializer instance +azure_credential = DefaultAzureCredential() +# create a AvroSerializer instance +avro_serializer = AvroSerializer( + client=SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=azure_credential + ), + group_name=GROUP_NAME, + auto_register_schemas=True +) + +async def send_event_data_batch(producer, serializer): + event_data_batch = await producer.create_batch() + dict_data = {"name": "Bob", "favorite_number": 7, "favorite_color": "red"} + # Use the serialize method to convert dict object to bytes with the given avro schema and set body of EventData. + # The serialize method will automatically register the schema into the Schema Registry Service and + # schema will be cached locally for future usage. + event_data = await serializer.serialize(data=dict_data, schema=SCHEMA_STRING, message_type=EventData) + print(f'The bytes of serialized dict data is {next(event_data.body)}.') + + event_data_batch.add(event_data) + await producer.send_batch(event_data_batch) + print('Send is done.') + + +async def main(): + + await send_event_data_batch(eventhub_producer, avro_serializer) + await avro_serializer.close() + await azure_credential.close() + await eventhub_producer.close() + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_event_data_message_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_event_data_message_async.py new file mode 100644 index 000000000000..83fde223f047 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_event_data_message_async.py @@ -0,0 +1,107 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os +import asyncio + +from azure.identity.aio import ClientSecretCredential +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer.aio import AvroSerializer +from azure.eventhub import EventData + +TENANT_ID=os.environ['AZURE_TENANT_ID'] +CLIENT_ID=os.environ['AZURE_CLIENT_ID'] +CLIENT_SECRET=os.environ['AZURE_CLIENT_SECRET'] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE=os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +GROUP_NAME=os.environ['SCHEMAREGISTRY_GROUP'] +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET +) + + +async def serialize_to_event_data_message(serializer): + dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + dict_data_alice = {"name": "Alice", "favorite_number": 15, "favorite_color": "green"} + + # Schema would be automatically registered into Schema Registry and cached locally. + event_data_ben = await serializer.serialize( + dict_data_ben, schema=SCHEMA_STRING, message_type=EventData + ) + + # The second call won't trigger a service call. + event_data_alice = await serializer.serialize( + dict_data_alice, schema=SCHEMA_STRING, message_type=EventData + ) + + print("Serialized data is: ", next(event_data_ben.body)) + print("Serialized data is: ", next(event_data_alice.body)) + + print("Serialized content type is: ", event_data_ben.content_type) + print("Serialized content type is: ", event_data_alice.content_type) + return [event_data_ben, event_data_alice] + + +async def deserialize_event_data_message(serializer, event_data): + # serializer.deserialize would extract the schema id from the content_type, + # retrieve schema from Schema Registry and cache the schema locally. + # If the schema id is in the local cache, the call won't trigger a service call. + deserialized_data = await serializer.deserialize(event_data) + + print("deserialized data is: ", deserialized_data) + return deserialized_data + + +async def main(): + schema_registry = SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=token_credential, + ) + serializer = AvroSerializer( + client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True + ) + event_data_ben, event_data_alice = await serialize_to_event_data_message(serializer) + deserialized_data_ben = await deserialize_event_data_message(serializer, event_data_ben) + deserialized_data_alice = await deserialize_event_data_message(serializer, event_data_alice) + await serializer.close() + await token_credential.close() + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_with_metadata_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_with_metadata_async.py new file mode 100644 index 000000000000..c10d8f33391b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_and_deserialize_with_metadata_async.py @@ -0,0 +1,97 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os +import asyncio + +from azure.identity.aio import ClientSecretCredential +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer.aio import AvroSerializer +from azure.eventhub import EventData + +TENANT_ID = os.environ["AZURE_TENANT_ID"] +CLIENT_ID = os.environ["AZURE_CLIENT_ID"] +CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ + "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" +] +GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET +) + + +async def serialize_metadata_dict(serializer): + dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + serialized_metadata_ben = await serializer.serialize(dict_data_ben, schema=SCHEMA_STRING) + + print("Serialized metadata is: ", serialized_metadata_ben) + return EventData.from_message_data( + serialized_metadata_ben["data"], + serialized_metadata_ben["content_type"], + ) + +async def deserialize_with_data_and_content_type(serializer, event_data): + # get data as bytes + data = bytearray() + for d in event_data.body: + data += d + data_bytes = bytes(data) + data_dict = {"data": data_bytes, "content_type": event_data.content_type} + deserialized_data = await serializer.deserialize(data_dict) + + print("deserialized data is: ", deserialized_data) + return deserialized_data + + +async def main(): + schema_registry = SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=token_credential, + ) + serializer = AvroSerializer( + client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True + ) + event_data = await serialize_metadata_dict(serializer) + deserialized_data = await deserialize_with_data_and_content_type(serializer, event_data) + await serializer.close() + await token_credential.close() + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_with_callback_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_with_callback_async.py new file mode 100644 index 000000000000..be0adc051ec0 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/async_samples/serialize_with_callback_async.py @@ -0,0 +1,88 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os +import asyncio + +from azure.identity.aio import ClientSecretCredential +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer.aio import AvroSerializer +from azure.eventhub import EventData + +TENANT_ID = os.environ["AZURE_TENANT_ID"] +CLIENT_ID = os.environ["AZURE_CLIENT_ID"] +CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ + "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" +] +GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET +) + +async def serialize_with_callback(serializer): + + # Callback MUST have these parameters in the following order: (data, content_type, **kwargs) + def sample_create_event_data(data, content_type, **kwargs): + print("Creating sample EventData with callback.") + return EventData.from_message_data(data, content_type, **kwargs) + + dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + event_data_ben = await serializer.serialize( + dict_data_ben, schema=SCHEMA_STRING, message_type=sample_create_event_data + ) + + print("Serialized data is: ", next(event_data_ben.body)) + print("Serialized content type is: ", event_data_ben.content_type) + return event_data_ben + +async def main(): + schema_registry = SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=token_credential, + ) + serializer = AvroSerializer( + client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True + ) + event_data_ben = await serialize_with_callback(serializer) + await serializer.close() + await token_credential.close() + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_receive_integration.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_receive_integration.py new file mode 100644 index 000000000000..502f01cc3817 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_receive_integration.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +Examples to show receiving events from EventHub with AvroSerializer integrated for data deserializing. +""" + +# pylint: disable=C0111 +import os +from azure.eventhub import EventHubConsumerClient +from azure.identity import DefaultAzureCredential +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer + +EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] +EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] + + +def on_event(partition_context, event): + print(f"Received event from partition: {partition_context.partition_id}.") + + bytes_payload = b"".join(b for b in event.body) + print(f'The received bytes of the EventData is {bytes_payload}.') + + # Use the deserialize method to deserialize the payload of the event. + # The deserialize method will extract the schema id from the content_type, and automatically retrieve the Avro Schema + # from the Schema Registry Service. The schema will be cached locally for future usage. + deserialized_data = avro_serializer.deserialize(event) + print(f'The dict data after deserializing is {deserialized_data}') + + +# create an EventHubConsumerClient instance +eventhub_consumer = EventHubConsumerClient.from_connection_string( + conn_str=EVENTHUB_CONNECTION_STR, + consumer_group='$Default', + eventhub_name=EVENTHUB_NAME, +) + + +# create a AvroSerializer instance +avro_serializer = AvroSerializer( + client=SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=DefaultAzureCredential() + ), + group_name=GROUP_NAME, + auto_register_schemas=True +) + + +try: + with eventhub_consumer, avro_serializer: + eventhub_consumer.receive( + on_event=on_event, + starting_position="-1", # "-1" is from the beginning of the partition. + ) +except KeyboardInterrupt: + print('Stopped receiving.') diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_send_integration.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_send_integration.py new file mode 100644 index 000000000000..023af1adc26b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/eventhub_send_integration.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +Examples to show sending event to EventHub with AvroSerializer integrated for data serializing. +""" + +# pylint: disable=C0111 + +import os +from azure.eventhub import EventHubProducerClient, EventData +from azure.identity import DefaultAzureCredential +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer + +EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] +EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] +GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] + +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +def send_event_data_batch(producer, serializer): + event_data_batch = producer.create_batch() + dict_data = {"name": "Bob", "favorite_number": 7, "favorite_color": "red"} + # Use the serialize method to convert dict object to bytes with the given avro schema and set body of EventData. + # The serialize method will automatically register the schema into the Schema Registry Service and + # schema will be cached locally for future usage. + event_data = serializer.serialize(data=dict_data, schema=SCHEMA_STRING, message_type=EventData) + print(f'The bytes of serialized dict data is {next(event_data.body)}.') + + event_data_batch.add(event_data) + producer.send_batch(event_data_batch) + print('Send is done.') + + +# create an EventHubProducerClient instance +eventhub_producer = EventHubProducerClient.from_connection_string( + conn_str=EVENTHUB_CONNECTION_STR, + eventhub_name=EVENTHUB_NAME +) + + +# create a AvroSerializer instance +avro_serializer = AvroSerializer( + client=SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=DefaultAzureCredential() + ), + group_name=GROUP_NAME, + auto_register_schemas=True +) + + +with eventhub_producer, avro_serializer: + send_event_data_batch(eventhub_producer, avro_serializer) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_event_data_message.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_event_data_message.py new file mode 100644 index 000000000000..a7aecaf3073d --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_event_data_message.py @@ -0,0 +1,101 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os + +from azure.identity import ClientSecretCredential +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.eventhub import EventData + +TENANT_ID = os.environ["AZURE_TENANT_ID"] +CLIENT_ID = os.environ["AZURE_CLIENT_ID"] +CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ + "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" +] +GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET +) + + +def serialize_to_event_data_message(serializer): + dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + dict_data_alice = {"name": "Alice", "favorite_number": 15, "favorite_color": "green"} + + # Schema would be automatically registered into Schema Registry and cached locally. + event_data_ben = serializer.serialize( + dict_data_ben, schema=SCHEMA_STRING, message_type=EventData + ) + + # The second call won't trigger a service call. + event_data_alice = serializer.serialize( + dict_data_alice, schema=SCHEMA_STRING, message_type=EventData + ) + + print("Serialized data is: ", next(event_data_ben.body)) + print("Serialized data is: ", next(event_data_alice.body)) + + print("Serialized content type is: ", event_data_ben.content_type) + print("Serialized content type is: ", event_data_alice.content_type) + return [event_data_ben, event_data_alice] + + +def deserialize_event_data_message(serializer, event_data): + # serializer.deserialize would extract the schema id from the content_type, + # retrieve schema from Schema Registry and cache the schema locally. + # If the schema id is in the local cache, the call won't trigger a service call. + deserialized_data = serializer.deserialize(event_data) + + print("Deserialized data is: ", deserialized_data) + return deserialized_data + + +if __name__ == "__main__": + schema_registry = SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=token_credential, + ) + serializer = AvroSerializer( + client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True + ) + event_data_ben, event_data_alice = serialize_to_event_data_message(serializer) + deserialized_data_ben = deserialize_event_data_message(serializer, event_data_ben) + deserialized_data_alice = deserialize_event_data_message(serializer, event_data_alice) + serializer.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_with_metadata.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_with_metadata.py new file mode 100644 index 000000000000..039d69469ca4 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_and_deserialize_with_metadata.py @@ -0,0 +1,91 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os + +from azure.identity import ClientSecretCredential +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.eventhub import EventData + +TENANT_ID = os.environ["AZURE_TENANT_ID"] +CLIENT_ID = os.environ["AZURE_CLIENT_ID"] +CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ + "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" +] +GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET +) + + +def serialize_metadata_dict(serializer): + dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + serialized_metadata_ben = serializer.serialize(dict_data_ben, schema=SCHEMA_STRING) + + print("Serialized metadata is: ", serialized_metadata_ben) + return EventData.from_message_data( + serialized_metadata_ben["data"], + serialized_metadata_ben["content_type"], + ) + +def deserialize_with_data_and_content_type(serializer, event_data): + # get data as bytes + data = bytearray() + for d in event_data.body: + data += d + data_bytes = bytes(data) + data_dict = {"data": data_bytes, "content_type": event_data.content_type} + deserialized_data = serializer.deserialize(data_dict) + + print("Deserialized data is: ", deserialized_data) + return deserialized_data + + +if __name__ == "__main__": + schema_registry = SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=token_credential, + ) + serializer = AvroSerializer( + client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True + ) + event_data = serialize_metadata_dict(serializer) + deserialized_data = deserialize_with_data_and_content_type(serializer, event_data) + serializer.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_with_callback.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_with_callback.py new file mode 100644 index 000000000000..f451c9312f9d --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/sync_samples/serialize_with_callback.py @@ -0,0 +1,83 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os + +from azure.identity import ClientSecretCredential +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.eventhub import EventData + +TENANT_ID = os.environ["AZURE_TENANT_ID"] +CLIENT_ID = os.environ["AZURE_CLIENT_ID"] +CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] + +SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ + "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" +] +GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] +SCHEMA_STRING = """ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +}""" + + +token_credential = ClientSecretCredential( + tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET +) + +def serialize_with_callback(serializer): + + # Callback MUST have these parameters in the following order: (data, content_type, **kwargs) + def sample_create_event_data(data, content_type, **kwargs): + print("Creating sample EventData with callback.") + return EventData.from_message_data(data, content_type, **kwargs) + + dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} + event_data_ben = serializer.serialize( + dict_data_ben, schema=SCHEMA_STRING, message_type=sample_create_event_data + ) + + print("Serialized data is: ", next(event_data_ben.body)) + print("Serialized content type is: ", event_data_ben.content_type) + return event_data_ben + + +if __name__ == "__main__": + schema_registry = SchemaRegistryClient( + fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, + credential=token_credential, + ) + serializer = AvroSerializer( + client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True + ) + event_data_ben = serialize_with_callback(serializer) + serializer.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/sdk_packaging.toml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/sdk_packaging.toml new file mode 100644 index 000000000000..e7687fdae93b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/sdk_packaging.toml @@ -0,0 +1,2 @@ +[packaging] +auto_update = false \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py new file mode 100644 index 000000000000..ddcedeefb8ad --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +import re +import sys +import os.path +from io import open +from setuptools import find_packages, setup + +# Change the PACKAGE_NAME only to change folder and different name +PACKAGE_NAME = "azure-schemaregistry-avroserializer" +PACKAGE_PPRINT_NAME = "Schema Registry Avro Serializer" + +package_folder_path = "azure/schemaregistry/serializer/avroserializer" +namespace_name = "azure.schemaregistry.serializer.avroserializer" + +# Version extraction inspired from 'requests' +with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', + fd.read(), re.MULTILINE).group(1) + +if not version: + raise RuntimeError('Cannot find version information') + +with open('README.md', encoding='utf-8') as f: + readme = f.read() +with open('CHANGELOG.md', encoding='utf-8') as f: + changelog = f.read() + +exclude_packages = [ + 'tests', + 'samples', + # Exclude packages that will be covered by PEP420 or nspkg + 'azure', + 'azure.schemaregistry', + ] +install_packages = [ + 'azure-schemaregistry>=1.0.0,<2.0.0', + 'avro>=1.11.0' +] + +setup( + name=PACKAGE_NAME, + version=version, + description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME), + long_description=readme + '\n\n' + changelog, + long_description_content_type='text/markdown', + license='MIT License', + author='Microsoft Corporation', + author_email='azpysdkhelp@microsoft.com', + url='https://github.com/Azure/azure-sdk-for-python', + classifiers=[ + "Development Status :: 4 - Beta", + 'Programming Language :: Python', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'License :: OSI Approved :: MIT License', + ], + python_requires=">=3.6", + zip_safe=False, + packages=find_packages(exclude=exclude_packages), + install_requires=install_packages +) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml new file mode 100644 index 000000000000..3911d936188e --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml @@ -0,0 +1,92 @@ +interactions: +- request: + body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '201' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:45 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + response: + body: + string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + content-type: + - application/json;serialization=Avro + date: + - Fri, 25 Feb 2022 00:47:46 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml new file mode 100644 index 000000000000..65e3279983f3 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '201' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:47 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": + [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, + {"type": ["string", "null"], "name": "favorite_color"}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '221' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: POST + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:48 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + response: + body: + string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + content-type: + - application/json;serialization=Avro + date: + - Fri, 25 Feb 2022 00:47:48 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml new file mode 100644 index 000000000000..da00b82c35b5 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '201' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: POST + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:50 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": + [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, + {"type": ["string", "null"], "name": "favorite_color"}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '221' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: POST + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:51 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + response: + body: + string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + content-type: + - application/json;serialization=Avro + date: + - Fri, 25 Feb 2022 00:47:51 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml new file mode 100644 index 000000000000..b21312924136 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml @@ -0,0 +1,94 @@ +interactions: +- request: + body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.error\",\n + \ \"type\":\"error\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n + \ }" + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '167' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.error.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:53 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 8abba19fc05649a1a534166fc5060a08 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8abba19fc05649a1a534166fc5060a08?api-version=2021-10 + schema-name: + - example.avro.error.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/8abba19fc05649a1a534166fc5060a08?api-version=2021-10 + response: + body: + string: '{"name":"User","namespace":"example.avro.error","type":"error","fields":[{"name":"name","type":"string"}]}' + headers: + content-type: + - application/json;serialization=Avro + date: + - Fri, 25 Feb 2022 00:47:54 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 8abba19fc05649a1a534166fc5060a08 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8abba19fc05649a1a534166fc5060a08?api-version=2021-10 + schema-name: + - example.avro.error.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml new file mode 100644 index 000000000000..b9c2981c5b3b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml @@ -0,0 +1,185 @@ +interactions: +- request: + body: "{\n \"namespace\": \"thrownaway\",\n \"name\":\"User.avro\",\n + \ \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n + \ }" + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '166' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User.avro?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:56 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a3411941a0456cac738c425338b1db + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a3411941a0456cac738c425338b1db?api-version=2021-10 + schema-name: + - User.avro + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/08a3411941a0456cac738c425338b1db?api-version=2021-10 + response: + body: + string: '{"namespace":"thrownaway","name":"User.avro","type":"record","fields":[{"name":"name","type":"string"}]}' + headers: + content-type: + - application/json;serialization=Avro + date: + - Fri, 25 Feb 2022 00:47:56 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a3411941a0456cac738c425338b1db + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a3411941a0456cac738c425338b1db?api-version=2021-10 + schema-name: + - User.avro + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 200 + message: OK +- request: + body: "{\n \"name\":\"User\",\n \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n + \ }" + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '122' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:57 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - e088feb3752c492883a129cb7664daa6 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/e088feb3752c492883a129cb7664daa6?api-version=2021-10 + schema-name: + - User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/e088feb3752c492883a129cb7664daa6?api-version=2021-10 + response: + body: + string: '{"name":"User","type":"record","fields":[{"name":"name","type":"string"}]}' + headers: + content-type: + - application/json;serialization=Avro + date: + - Fri, 25 Feb 2022 00:47:57 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - e088feb3752c492883a129cb7664daa6 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/e088feb3752c492883a129cb7664daa6?api-version=2021-10 + schema-name: + - User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml new file mode 100644 index 000000000000..48ed1fec9c36 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: '{"type": "null"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '16' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/null?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:47:59 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/null/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 3dedb9f5663943e4848dfa778c8ea9aa + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/3dedb9f5663943e4848dfa778c8ea9aa?api-version=2021-10 + schema-name: + - 'null' + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/null/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml new file mode 100644 index 000000000000..d6cf017653f9 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml @@ -0,0 +1,96 @@ +interactions: +- request: + body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.populatedrecord\",\n + \ \"type\":\"record\",\n \"fields\":[\n {\"name\":\"name\",\"type\":\"string\"},\n + \ {\"name\":\"age\",\"type\":\"int\"},\n {\"name\":\"married\",\"type\":\"boolean\"},\n + \ {\"name\":\"height\",\"type\":\"float\"},\n {\"name\":\"randb\",\"type\":\"bytes\"}\n + \ ]\n }" + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '405' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.populatedrecord.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 00:48:02 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - f024ca12f4254593a89831cefc5c9e80 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/f024ca12f4254593a89831cefc5c9e80?api-version=2021-10 + schema-name: + - example.avro.populatedrecord.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/f024ca12f4254593a89831cefc5c9e80?api-version=2021-10 + response: + body: + string: '{"name":"User","namespace":"example.avro.populatedrecord","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"},{"name":"married","type":"boolean"},{"name":"height","type":"float"},{"name":"randb","type":"bytes"}]}' + headers: + content-type: + - application/json;serialization=Avro + date: + - Fri, 25 Feb 2022 00:48:02 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - f024ca12f4254593a89831cefc5c9e80 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/f024ca12f4254593a89831cefc5c9e80?api-version=2021-10 + schema-name: + - example.avro.populatedrecord.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml new file mode 100644 index 000000000000..95e2a7d6ebca --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml @@ -0,0 +1,63 @@ +interactions: +- request: + body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + Accept: + - application/json + Content-Length: + - '201' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:09 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + response: + body: + string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + content-type: application/json;serialization=Avro + date: Fri, 25 Feb 2022 00:51:09 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 200 + message: OK + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml new file mode 100644 index 000000000000..480b6ff5d3eb --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml @@ -0,0 +1,99 @@ +interactions: +- request: + body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": + [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, + {"type": ["string", "null"], "name": "favorite_color"}]}' + headers: + Accept: + - application/json + Content-Length: + - '221' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:10 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 +- request: + body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": + [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, + {"type": ["string", "null"], "name": "favorite_color"}]}' + headers: + Accept: + - application/json + Content-Length: + - '221' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: POST + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:10 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User:get-id?api-version=2021-10 +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + response: + body: + string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + content-type: application/json;serialization=Avro + date: Fri, 25 Feb 2022 00:51:11 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 200 + message: OK + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml new file mode 100644 index 000000000000..97fdeb34895b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml @@ -0,0 +1,99 @@ +interactions: +- request: + body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": + [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, + {"type": ["string", "null"], "name": "favorite_color"}]}' + headers: + Accept: + - application/json + Content-Length: + - '221' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:11 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 +- request: + body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": + [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, + {"type": ["string", "null"], "name": "favorite_color"}]}' + headers: + Accept: + - application/json + Content-Length: + - '221' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: POST + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:12 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User:get-id?api-version=2021-10 +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + response: + body: + string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + content-type: application/json;serialization=Avro + date: Fri, 25 Feb 2022 00:51:12 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 200 + message: OK + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml new file mode 100644 index 000000000000..8b4c7335af5d --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml @@ -0,0 +1,65 @@ +interactions: +- request: + body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.error\",\n + \ \"type\":\"error\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n + \ }" + headers: + Accept: + - application/json + Content-Length: + - '167' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.error.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:14 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 8abba19fc05649a1a534166fc5060a08 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8abba19fc05649a1a534166fc5060a08?api-version=2021-10 + schema-name: example.avro.error.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.error.User?api-version=2021-10 +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/8abba19fc05649a1a534166fc5060a08?api-version=2021-10 + response: + body: + string: '{"name":"User","namespace":"example.avro.error","type":"error","fields":[{"name":"name","type":"string"}]}' + headers: + content-type: application/json;serialization=Avro + date: Fri, 25 Feb 2022 00:51:14 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 8abba19fc05649a1a534166fc5060a08 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8abba19fc05649a1a534166fc5060a08?api-version=2021-10 + schema-name: example.avro.error.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 200 + message: OK + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/8abba19fc05649a1a534166fc5060a08?api-version=2021-10 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml new file mode 100644 index 000000000000..9a2d9aa4ec58 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml @@ -0,0 +1,127 @@ +interactions: +- request: + body: "{\n \"namespace\": \"thrownaway\",\n \"name\":\"User.avro\",\n + \ \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n + \ }" + headers: + Accept: + - application/json + Content-Length: + - '166' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User.avro?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:15 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a3411941a0456cac738c425338b1db + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a3411941a0456cac738c425338b1db?api-version=2021-10 + schema-name: User.avro + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/User.avro?api-version=2021-10 +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/08a3411941a0456cac738c425338b1db?api-version=2021-10 + response: + body: + string: '{"namespace":"thrownaway","name":"User.avro","type":"record","fields":[{"name":"name","type":"string"}]}' + headers: + content-type: application/json;serialization=Avro + date: Fri, 25 Feb 2022 00:51:16 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a3411941a0456cac738c425338b1db + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a3411941a0456cac738c425338b1db?api-version=2021-10 + schema-name: User.avro + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 200 + message: OK + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/08a3411941a0456cac738c425338b1db?api-version=2021-10 +- request: + body: "{\n \"name\":\"User\",\n \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n + \ }" + headers: + Accept: + - application/json + Content-Length: + - '122' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:16 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: e088feb3752c492883a129cb7664daa6 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/e088feb3752c492883a129cb7664daa6?api-version=2021-10 + schema-name: User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/User?api-version=2021-10 +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/e088feb3752c492883a129cb7664daa6?api-version=2021-10 + response: + body: + string: '{"name":"User","type":"record","fields":[{"name":"name","type":"string"}]}' + headers: + content-type: application/json;serialization=Avro + date: Fri, 25 Feb 2022 00:51:17 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: e088feb3752c492883a129cb7664daa6 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/e088feb3752c492883a129cb7664daa6?api-version=2021-10 + schema-name: User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 200 + message: OK + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/e088feb3752c492883a129cb7664daa6?api-version=2021-10 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml new file mode 100644 index 000000000000..f2cfcc29b03c --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml @@ -0,0 +1,34 @@ +interactions: +- request: + body: '{"type": "null"}' + headers: + Accept: + - application/json + Content-Length: + - '16' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/null?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:18 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/null/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 3dedb9f5663943e4848dfa778c8ea9aa + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/3dedb9f5663943e4848dfa778c8ea9aa?api-version=2021-10 + schema-name: 'null' + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/null/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/null?api-version=2021-10 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml new file mode 100644 index 000000000000..0a3099c6b67b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.populatedrecord\",\n + \ \"type\":\"record\",\n \"fields\":[\n {\"name\":\"name\",\"type\":\"string\"},\n + \ {\"name\":\"age\",\"type\":\"int\"},\n {\"name\":\"married\",\"type\":\"boolean\"},\n + \ {\"name\":\"height\",\"type\":\"float\"},\n {\"name\":\"randb\",\"type\":\"bytes\"}\n + \ ]\n }" + headers: + Accept: + - application/json + Content-Length: + - '405' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.populatedrecord.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 00:51:20 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: f024ca12f4254593a89831cefc5c9e80 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/f024ca12f4254593a89831cefc5c9e80?api-version=2021-10 + schema-name: example.avro.populatedrecord.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.populatedrecord.User?api-version=2021-10 +- request: + body: null + headers: + Accept: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: GET + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/f024ca12f4254593a89831cefc5c9e80?api-version=2021-10 + response: + body: + string: '{"name":"User","namespace":"example.avro.populatedrecord","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"},{"name":"married","type":"boolean"},{"name":"height","type":"float"},{"name":"randb","type":"bytes"}]}' + headers: + content-type: application/json;serialization=Avro + date: Fri, 25 Feb 2022 00:51:20 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: f024ca12f4254593a89831cefc5c9e80 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/f024ca12f4254593a89831cefc5c9e80?api-version=2021-10 + schema-name: example.avro.populatedrecord.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + transfer-encoding: chunked + status: + code: 200 + message: OK + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/f024ca12f4254593a89831cefc5c9e80?api-version=2021-10 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py new file mode 100644 index 000000000000..5542943a1106 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -0,0 +1,500 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import functools +import pytest +import json + +from azure.schemaregistry import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer import AvroSerializer +from azure.schemaregistry.serializer.avroserializer.exceptions import SchemaParseError, SchemaSerializationError, SchemaDeserializationError + +import avro +from avro.errors import AvroTypeException +from azure.schemaregistry.serializer.avroserializer._apache_avro_serializer import ApacheAvroObjectSerializer as AvroObjectSerializer + +from devtools_testutils import AzureTestCase, PowerShellPreparer + +SchemaRegistryPowerShellPreparer = functools.partial(PowerShellPreparer, "schemaregistry", schemaregistry_fully_qualified_namespace="fake_resource.servicebus.windows.net/", schemaregistry_group="fakegroup") + +class AvroSerializerTests(AzureTestCase): + + def test_raw_avro_serializer(self): + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + + raw_avro_object_serializer = AvroObjectSerializer() + + # serializing part + serialized_payload = raw_avro_object_serializer.serialize(dict_data, schema_str) + + # deserializing part + deserialized_data = raw_avro_object_serializer.deserialize(serialized_payload, schema_str) + + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + assert deserialized_data["favorite_color"] == u"red" + + dict_data_missing_optional_fields = {"name": u"Alice"} + serialized_payload = raw_avro_object_serializer.serialize(dict_data_missing_optional_fields, schema_str) + deserialized_data = raw_avro_object_serializer.deserialize(serialized_payload, schema_str) + + assert deserialized_data["name"] == u"Alice" + assert not deserialized_data["favorite_number"] + assert not deserialized_data["favorite_color"] + + def test_raw_avro_serializer_negative(self): + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + + raw_avro_object_serializer = AvroObjectSerializer() + dict_data_wrong_type = {"name": u"Ben", "favorite_number": u"something", "favorite_color": u"red"} + with pytest.raises(AvroTypeException): # avro.io.AvroTypeException + raw_avro_object_serializer.serialize(dict_data_wrong_type, schema_str) + + dict_data_missing_required_field = {"favorite_number": 7, "favorite_color": u"red"} + with pytest.raises(AvroTypeException): # avro.io.AvroTypeException + raw_avro_object_serializer.serialize(dict_data_missing_required_field, schema_str) + + @SchemaRegistryPowerShellPreparer() + def test_basic_sr_avro_serializer_with_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + serialized_metadata = sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + + assert content_type.split("+")[0] == 'avro/binary' + schema_id = sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro").id + assert content_type.split("+")[1] == schema_id + + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = sr_avro_serializer.deserialize(serialized_data_dict) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + assert deserialized_data["favorite_color"] == u"red" + + # check that AvroSerializer won't work with message types that don't follow protocols + class BadExample: + def __init__(self, not_data): + self.not_data = not_data + + with pytest.raises(SchemaSerializationError) as e: # caught avro SchemaParseError + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_str, message_type=BadExample) + assert "subtype of the MessageType" in (str(e.value)) + + bad_ex = BadExample('fake') + with pytest.raises(SchemaDeserializationError) as e: # caught avro SchemaParseError + sr_avro_serializer.deserialize(message=bad_ex) + assert "subtype of the MessageType" in (str(e.value)) + + # check that AvroSerializer will work with message types that follow protocols + class GoodExample: + def __init__(self, data: bytes, content_type: str, **kwargs): + self.data = data + self.content_type = content_type + self.extra = kwargs.pop('extra', None) + + def __message_data__(self): + return {"data": self.data, "content_type": self.content_type} + + def good_callback(data: bytes, content_type: str, **kwargs): + return GoodExample(data, content_type, **kwargs) + + good_ex_obj = sr_avro_serializer.serialize(dict_data, schema=schema_str, message_type=GoodExample, extra='val') + good_ex_callback = sr_avro_serializer.serialize(dict_data, schema=schema_str, message_type=good_callback, extra='val') + deserialized_data_obj = sr_avro_serializer.deserialize(message=good_ex_obj) + deserialized_data_callback = sr_avro_serializer.deserialize(message=good_ex_callback) + + assert deserialized_data_obj["name"] == u"Ben" + assert deserialized_data_obj["favorite_number"] == 7 + assert deserialized_data_obj["favorite_color"] == u"red" + assert deserialized_data_callback["name"] == u"Ben" + assert deserialized_data_callback["favorite_number"] == 7 + assert deserialized_data_callback["favorite_color"] == u"red" + + sr_avro_serializer.close() + + @SchemaRegistryPowerShellPreparer() + def test_basic_sr_avro_serializer_without_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group) + + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + serialized_metadata = sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + + assert content_type.split("+")[0] == 'avro/binary' + schema_id = sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro").id + assert content_type.split("+")[1] == schema_id + + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = sr_avro_serializer.deserialize(serialized_data_dict) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + assert deserialized_data["favorite_color"] == u"red" + + sr_avro_serializer.close() + + @SchemaRegistryPowerShellPreparer() + def test_basic_sr_avro_serializer_deserialize_readers_schema(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + serialized_metadata = sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + + # readers_schema with removed field + readers_schema_remove_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]}]}""" + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_remove_field) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + + # readers_schema with extra field with default + readers_schema_extra_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}, {"name":"favorite_city","type":["string","null"], "default": "Redmond"}]}""" + deserialized_data = sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_extra_field) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + assert deserialized_data["favorite_color"] == "red" + assert deserialized_data["favorite_city"] == "Redmond" + + # readers_schema with changed name results in error + readers_schema_change_name = """{"namespace":"fakeexample.avro","type":"record","name":"fake_user","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + with pytest.raises(SchemaDeserializationError): + deserialized_data = sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_change_name) + + + ################################################################# + ######################### PARSE SCHEMAS ######################### + ################################################################# + + @SchemaRegistryPowerShellPreparer() + def test_parse_invalid_json_string(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + invalid_schema = { + "name":"User", + "type":"record", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + } + invalid_schema_string = "{}".format(invalid_schema) + with pytest.raises(SchemaParseError): # caught avro SchemaParseError + sr_avro_serializer.serialize({"name": u"Ben"}, schema=invalid_schema_string) + + ######################### PRIMITIVES ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_primitive_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + primitive_string = "string" + with pytest.raises(SchemaParseError) as e: + sr_avro_serializer.serialize("hello", schema=primitive_string) + + ######################### type fixed ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_fixed_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + # avro bug: should give warning from IgnoredLogicalType error since precision < 0 + #fixed_type_ignore_logical_type_error = """{"type": "fixed", "size": 4, "namespace":"example.avro", "name":"User", "precision": -1}""" + #sr_avro_serializer.serialize({}, schema=fixed_type_ignore_logical_type_error) + + schema_no_size = """{"type": "fixed", "name":"User"}""" + with pytest.raises(SchemaParseError): # caught AvroException + sr_avro_serializer.serialize({}, schema=schema_no_size) + + schema_no_name = """{"type": "fixed", "size": 3}""" + with pytest.raises(SchemaParseError): # caught SchemaParseError + sr_avro_serializer.serialize({}, schema=schema_no_name) + + schema_wrong_name = """{"type": "fixed", "name": 1, "size": 3}""" + with pytest.raises(SchemaParseError): # caught SchemaParseError + sr_avro_serializer.serialize({}, schema=schema_wrong_name) + + schema_wrong_namespace = """{"type": "fixed", "name": "User", "size": 3, "namespace": 1}""" + with pytest.raises(SchemaParseError): # caught SchemaParseError + sr_avro_serializer.serialize({}, schema=schema_wrong_namespace) + + ######################### type unspecified ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_invalid_type(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_no_type = """{ + "name": "User", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): # caught avro SchemaParseError + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_type) + + schema_wrong_type_type = """{ + "name":"User", + "type":1, + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_type) + + ######################### RECORD SCHEMA ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_record_name(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_name_has_dot = """{ + "namespace": "thrownaway", + "name":"User.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + serialized_schema = sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_name_has_dot) + schema_id = serialized_schema["content_type"].split("+")[1] + registered_schema = sr_client.get_schema(schema_id) + deserialized_registered_schema = json.loads(registered_schema.definition) + + # ensure that namespace is saved as part of name before . in registered schema + assert deserialized_registered_schema["name"] == "User.avro" + assert deserialized_registered_schema["namespace"] == "thrownaway" + + schema_name_no_namespace = """{ + "name":"User", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + serialized_schema = sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_name_no_namespace) + schema_id = serialized_schema["content_type"].split("+")[1] + registered_schema = sr_client.get_schema(schema_id) + deserialized_registered_schema = json.loads(registered_schema.definition) + + assert deserialized_registered_schema["name"] == "User" + assert "namespace" not in deserialized_registered_schema + + schema_invalid_fullname = """{ + "name":"abc", + "type":"record", + "namespace":"9example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_fullname) + + schema_invalid_name_in_fullname = """{ + "name":"1abc", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_name_in_fullname) + + schema_invalid_name_reserved_type = """{ + "name":"record", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_name_reserved_type) + + schema_wrong_type_name = """{ + "name":1, + "type":"record", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_name) + + schema_no_name = """{ + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_name) + + @SchemaRegistryPowerShellPreparer() + def test_parse_error_schema_as_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_error_type = """{ + "name":"User", + "namespace":"example.avro.error", + "type":"error", + "fields":[{"name":"name","type":"string"}] + }""" + serialized_metadata = sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_error_type) + schema_id = serialized_metadata["content_type"].split("+")[1] + registered_schema = sr_client.get_schema(schema_id) + deserialized_registered_schema = json.loads(registered_schema.definition) + assert deserialized_registered_schema["type"] == "error" + + @SchemaRegistryPowerShellPreparer() + def test_parse_record_fields(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_no_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_fields) + + schema_wrong_type_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + "fields": "hello" + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_fields) + + schema_wrong_field_item_type = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + "fields": ["hello"] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_field_item_type) + + schema_record_field_no_name= """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_no_name) + + schema_record_field_wrong_type_name= """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name": 1, "type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_wrong_type_name) + + schema_record_field_with_invalid_order = """{ + "name":"User", + "namespace":"example.avro.order", + "type":"record", + "fields":[{"name":"name","type":"string","order":"fake_order"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_with_invalid_order) + + schema_record_duplicate_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}, {"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_duplicate_fields) + + schema_field_type_invalid = """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":1}] + }""" + with pytest.raises(SchemaParseError): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_field_type_invalid) + + ################################################################# + #################### SERIALIZE AND DESERIALIZE ################## + ################################################################# + + @SchemaRegistryPowerShellPreparer() + def test_serialize_primitive(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + null_type = """{"type": "null"}""" + serialized_metadata = sr_avro_serializer.serialize(None, schema=null_type) + assert len(serialized_metadata["data"]) == 0 # assert no data serialized + + @SchemaRegistryPowerShellPreparer() + def test_serialize_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + # add below to schema fields later if needed + # {"name":"example.innerrec","type":"record","fields":[{"name":"a","type":"int"}]}, + # {"name":"innerenum","type":"enum","symbols":["FOO", "BAR"]}, + # {"name":"innerarray","type":"array","items":"int"}, + # {"name":"innermap","type":"map","values":"int"}, + # {"name":"innerfixed","type":"fixed","size":74} + schema_record = """{ + "name":"User", + "namespace":"example.avro.populatedrecord", + "type":"record", + "fields":[ + {"name":"name","type":"string"}, + {"name":"age","type":"int"}, + {"name":"married","type":"boolean"}, + {"name":"height","type":"float"}, + {"name":"randb","type":"bytes"} + ] + }""" + data = { + "name": u"Ben", + "age": 3, + "married": False, + "height": 13.5, + "randb": b"\u00FF" + } + + serialized_metadata = sr_avro_serializer.serialize(data, schema=schema_record) + deserialized_data = sr_avro_serializer.deserialize(serialized_metadata) + assert deserialized_data == data diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer_async.py new file mode 100644 index 000000000000..e74eb98b8973 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer_async.py @@ -0,0 +1,476 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import functools +import pytest +import uuid +import json +from io import BytesIO +import pytest + +import avro +from avro.errors import AvroTypeException + + +from azure.schemaregistry.aio import SchemaRegistryClient +from azure.schemaregistry.serializer.avroserializer.aio import AvroSerializer +from azure.schemaregistry.serializer.avroserializer.exceptions import SchemaParseError, SchemaSerializationError, SchemaDeserializationError + +from devtools_testutils import AzureTestCase, PowerShellPreparer + +SchemaRegistryPowerShellPreparer = functools.partial(PowerShellPreparer, "schemaregistry", schemaregistry_fully_qualified_namespace="fake_resource.servicebus.windows.net/", schemaregistry_group="fakegroup") + +class AvroSerializerAsyncTests(AzureTestCase): + + def create_client(self, fully_qualified_namespace): + credential = self.get_credential(SchemaRegistryClient, is_async=True) + return self.create_client_from_credential(SchemaRegistryClient, credential, fully_qualified_namespace=fully_qualified_namespace, is_async=True) + + @pytest.mark.asyncio + @SchemaRegistryPowerShellPreparer() + async def test_basic_sr_avro_serializer_with_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + async with sr_client: + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema_str = "{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], \"name\": \"favorite_color\"}]}" + schema = avro.schema.parse(schema_str) + + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + serialized_metadata = await sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + + assert content_type.split("+")[0] == 'avro/binary' + schema_properties = await sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro") + schema_id = schema_properties.id + assert content_type.split("+")[1] == schema_id + + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + assert deserialized_data["favorite_color"] == u"red" + + # check that AvroSerializer won't work with message types that don't follow protocols + class BadExample: + def __init__(self, not_data): + self.not_data = not_data + + with pytest.raises(SchemaSerializationError) as e: # caught avro SchemaParseError + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_str, message_type=BadExample) + assert "subtype of the MessageType" in (str(e.value)) + + bad_ex = BadExample('fake') + with pytest.raises(SchemaDeserializationError) as e: # caught avro SchemaParseError + await sr_avro_serializer.deserialize(message=bad_ex) + assert "subtype of the MessageType" in (str(e.value)) + + # check that AvroSerializer will work with message types that follow protocols + class GoodExample: + def __init__(self, data: bytes, content_type: str, **kwargs): + self.data = data + self.content_type = content_type + self.extra = kwargs.pop('extra', None) + + def __message_data__(self): + return {"data": self.data, "content_type": self.content_type} + + def good_callback(data: bytes, content_type: str, **kwargs): + return GoodExample(data, content_type, **kwargs) + + good_ex_obj = await sr_avro_serializer.serialize(dict_data, schema=schema_str, message_type=GoodExample, extra='val') + good_ex_callback = await sr_avro_serializer.serialize(dict_data, schema=schema_str, message_type=good_callback, extra='val') + deserialized_data_obj = await sr_avro_serializer.deserialize(message=good_ex_obj) + deserialized_data_callback = await sr_avro_serializer.deserialize(message=good_ex_callback) + + assert deserialized_data_obj["name"] == u"Ben" + assert deserialized_data_obj["favorite_number"] == 7 + assert deserialized_data_obj["favorite_color"] == u"red" + assert deserialized_data_callback["name"] == u"Ben" + assert deserialized_data_callback["favorite_number"] == 7 + assert deserialized_data_callback["favorite_color"] == u"red" + + @pytest.mark.asyncio + @SchemaRegistryPowerShellPreparer() + async def test_basic_sr_avro_serializer_without_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + async with sr_client: + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema_str = "{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], \"name\": \"favorite_color\"}]}" + schema = avro.schema.parse(schema_str) + + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + serialized_metadata = await sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + + assert content_type.split("+")[0] == 'avro/binary' + schema_properties = await sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro") + schema_id = schema_properties.id + assert content_type.split("+")[1] == schema_id + + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + assert deserialized_data["favorite_color"] == u"red" + + @pytest.mark.asyncio + @SchemaRegistryPowerShellPreparer() + async def test_basic_sr_avro_serializer_deserialize_readers_schema(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + serialized_metadata = await sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + + # readers_schema with removed field + readers_schema_remove_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]}]}""" + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_remove_field) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + + # readers_schema with extra field with default + readers_schema_extra_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}, {"name":"favorite_city","type":["string","null"], "default": "Redmond"}]}""" + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_extra_field) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + assert deserialized_data["favorite_color"] == "red" + assert deserialized_data["favorite_city"] == "Redmond" + + # readers_schema with changed name results in error + readers_schema_change_name = """{"namespace":"fakeexample.avro","type":"record","name":"fake_user","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + with pytest.raises(SchemaDeserializationError): + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_change_name) + print(deserialized_data) + + + ################################################################# + ######################### PARSE SCHEMAS ######################### + ################################################################# + + @SchemaRegistryPowerShellPreparer() + async def test_parse_invalid_json_string(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + invalid_schema = { + "name":"User", + "type":"record", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + } + invalid_schema_string = "{}".format(invalid_schema) + with pytest.raises(SchemaParseError): # caught avro SchemaParseError + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=invalid_schema_string) + + ######################### PRIMITIVES ######################### + + @SchemaRegistryPowerShellPreparer() + async def test_parse_primitive_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + primitive_string = "string" + with pytest.raises(SchemaParseError) as e: + await sr_avro_serializer.serialize("hello", schema=primitive_string) + + ######################### type fixed ######################### + + @SchemaRegistryPowerShellPreparer() + async def test_parse_fixed_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + # avro bug: should give warning from IgnoredLogicalType error since precision < 0 + #fixed_type_ignore_logical_type_error = """{"type": "fixed", "size": 4, "namespace":"example.avro", "name":"User", "precision": -1}""" + #await sr_avro_serializer.serialize({}, schema=fixed_type_ignore_logical_type_error) + + schema_no_size = """{"type": "fixed", "name":"User"}""" + with pytest.raises(SchemaParseError): # caught AvroException + await sr_avro_serializer.serialize({}, schema=schema_no_size) + + schema_no_name = """{"type": "fixed", "size": 3}""" + with pytest.raises(SchemaParseError): # caught SchemaParseError + await sr_avro_serializer.serialize({}, schema=schema_no_name) + + schema_wrong_name = """{"type": "fixed", "name": 1, "size": 3}""" + with pytest.raises(SchemaParseError): # caught SchemaParseError + await sr_avro_serializer.serialize({}, schema=schema_wrong_name) + + schema_wrong_namespace = """{"type": "fixed", "name": "User", "size": 3, "namespace": 1}""" + with pytest.raises(SchemaParseError): # caught SchemaParseError + await sr_avro_serializer.serialize({}, schema=schema_wrong_namespace) + + ######################### type unspecified ######################### + + @SchemaRegistryPowerShellPreparer() + async def test_parse_invalid_type(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_no_type = """{ + "name": "User", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): # caught avro SchemaParseError + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_type) + + schema_wrong_type_type = """{ + "name":"User", + "type":1, + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_type) + + ######################### RECORD SCHEMA ######################### + + @SchemaRegistryPowerShellPreparer() + async def test_parse_record_name(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_name_has_dot = """{ + "namespace": "thrownaway", + "name":"User.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + serialized_schema = await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_name_has_dot) + schema_id = serialized_schema["content_type"].split("+")[1] + registered_schema = await sr_client.get_schema(schema_id) + deserialized_registered_schema = json.loads(registered_schema.definition) + + assert deserialized_registered_schema["name"] == "User.avro" + assert deserialized_registered_schema["namespace"] == "thrownaway" + + schema_name_no_namespace = """{ + "name":"User", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + serialized_schema = await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_name_no_namespace) + schema_id = serialized_schema["content_type"].split("+")[1] + registered_schema = await sr_client.get_schema(schema_id) + deserialized_registered_schema = json.loads(registered_schema.definition) + + assert deserialized_registered_schema["name"] == "User" + assert "namespace" not in deserialized_registered_schema + + schema_invalid_fullname = """{ + "name":"abc", + "type":"record", + "namespace":"9example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_fullname) + + schema_invalid_name_in_fullname = """{ + "name":"1abc", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_name_in_fullname) + + schema_invalid_name_reserved_type = """{ + "name":"record", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_name_reserved_type) + + schema_wrong_type_name = """{ + "name":1, + "type":"record", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_name) + + schema_no_name = """{ + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_name) + + @SchemaRegistryPowerShellPreparer() + async def test_parse_error_schema_as_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_error_type = """{ + "name":"User", + "namespace":"example.avro.error", + "type":"error", + "fields":[{"name":"name","type":"string"}] + }""" + serialized_metadata = await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_error_type) + schema_id = serialized_metadata["content_type"].split("+")[1] + registered_schema = await sr_client.get_schema(schema_id) + deserialized_registered_schema = json.loads(registered_schema.definition) + assert deserialized_registered_schema["type"] == "error" + + @SchemaRegistryPowerShellPreparer() + async def test_parse_record_fields(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_no_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_fields) + + schema_wrong_type_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + "fields": "hello" + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_fields) + + schema_wrong_field_item_type = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + "fields": ["hello"] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_field_item_type) + + schema_record_field_no_name= """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_no_name) + + schema_record_field_wrong_type_name= """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name": 1, "type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_wrong_type_name) + + schema_record_field_with_invalid_order = """{ + "name":"User", + "namespace":"example.avro.order", + "type":"record", + "fields":[{"name":"name","type":"string","order":"fake_order"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_with_invalid_order) + + schema_record_duplicate_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}, {"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_duplicate_fields) + + schema_field_type_invalid = """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":1}] + }""" + with pytest.raises(SchemaParseError): + await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_field_type_invalid) + + ################################################################# + #################### serializE AND deserializE ########################## + ################################################################# + + @SchemaRegistryPowerShellPreparer() + async def test_serialize_primitive(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + null_type = """{"type": "null"}""" + serialized_metadata = await sr_avro_serializer.serialize(None, schema=null_type) + assert len(serialized_metadata["data"]) == 0 # assert no data serialized + + @SchemaRegistryPowerShellPreparer() + async def test_serialize_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + # add below to schema later if possible + # {"name":"example.innerrec","type":"record","fields":[{"name":"a","type":"int"}]}, + # {"name":"innerenum","type":"enum","symbols":["FOO", "BAR"]}, + # {"name":"innerarray","type":"array","items":"int"}, + # {"name":"innermap","type":"map","values":"int"}, + # {"name":"innerfixed","type":"fixed","size":74} + schema_record = """{ + "name":"User", + "namespace":"example.avro.populatedrecord", + "type":"record", + "fields":[ + {"name":"name","type":"string"}, + {"name":"age","type":"int"}, + {"name":"married","type":"boolean"}, + {"name":"height","type":"float"}, + {"name":"randb","type":"bytes"} + ] + }""" + data = { + "name": u"Ben", + "age": 3, + "married": False, + "height": 13.5, + "randb": b"\u00FF" + } + + serialized_metadata = await sr_avro_serializer.serialize(data, schema=schema_record) + deserialized_data = await sr_avro_serializer.deserialize(serialized_metadata) + assert deserialized_data == data From 49cd72c2a9a7063b1bd014ee9c099a61796f86a3 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 25 Feb 2022 08:46:20 -0800 Subject: [PATCH 2/5] remove encoder --- .../CHANGELOG.md | 17 - .../azure-schemaregistry-avroencoder/LICENSE | 21 - .../MANIFEST.in | 6 - .../README.md | 329 ------------ .../azure/__init__.py | 26 - .../azure/schemaregistry/__init__.py | 26 - .../azure/schemaregistry/encoder/__init__.py | 26 - .../encoder/avroencoder/__init__.py | 37 -- .../avroencoder/_abstract_avro_encoder.py | 68 --- .../avroencoder/_apache_avro_encoder.py | 100 ---- .../encoder/avroencoder/_constants.py | 40 -- .../encoder/avroencoder/_message_protocol.py | 39 -- .../_schema_registry_avro_encoder.py | 285 ---------- .../encoder/avroencoder/_version.py | 27 - .../encoder/avroencoder/aio/__init__.py | 30 -- .../encoder/avroencoder/aio/_async_lru.py | 230 -------- .../_schema_registry_avro_encoder_async.py | 277 ---------- .../encoder/avroencoder/exceptions.py | 69 --- .../dev_requirements.txt | 4 - .../azure-schemaregistry-avroencoder/mypy.ini | 6 - .../samples/README.md | 65 --- ...ode_and_decode_event_data_message_async.py | 107 ---- .../encode_and_decode_with_metadata_async.py | 97 ---- .../encode_with_callback_async.py | 88 --- .../eventhub_receive_integration_async.py | 73 --- .../eventhub_send_integration_async.py | 79 --- .../encode_and_decode_event_data_message.py | 101 ---- .../encode_and_decode_with_metadata.py | 91 ---- .../sync_samples/encode_with_callback.py | 83 --- .../eventhub_receive_integration.py | 65 --- .../sync_samples/eventhub_send_integration.py | 71 --- .../sdk_packaging.toml | 2 - .../azure-schemaregistry-avroencoder/setup.py | 72 --- ...sr_avro_encoder_decode_readers_schema.yaml | 92 ---- ...ro_encoder_with_auto_register_schemas.yaml | 140 ----- ...encoder_without_auto_register_schemas.yaml | 140 ----- ...st_avro_encoder.test_encode_primitive.yaml | 48 -- .../test_avro_encoder.test_encode_record.yaml | 96 ---- ...der.test_parse_error_schema_as_record.yaml | 94 ---- ...t_avro_encoder.test_parse_record_name.yaml | 185 ------- ...sr_avro_encoder_decode_readers_schema.yaml | 63 --- ...ro_encoder_with_auto_register_schemas.yaml | 99 ---- ...encoder_without_auto_register_schemas.yaml | 99 ---- ...o_encoder_async.test_encode_primitive.yaml | 34 -- ...avro_encoder_async.test_encode_record.yaml | 67 --- ...ync.test_parse_error_schema_as_record.yaml | 65 --- ..._encoder_async.test_parse_record_name.yaml | 127 ----- ...serializer_with_auto_register_schemas.yaml | 140 ----- ...ializer_without_auto_register_schemas.yaml | 140 ----- ...zer.test_parse_error_schema_as_record.yaml | 94 ---- ...vro_serializer.test_parse_record_name.yaml | 185 ------- ...o_serializer.test_serialize_primitive.yaml | 48 -- ...avro_serializer.test_serialize_record.yaml | 96 ---- ...serializer_with_auto_register_schemas.yaml | 99 ---- ...ializer_without_auto_register_schemas.yaml | 99 ---- ...ync.test_parse_error_schema_as_record.yaml | 65 --- ...rializer_async.test_parse_record_name.yaml | 127 ----- ...alizer_async.test_serialize_primitive.yaml | 34 -- ...erializer_async.test_serialize_record.yaml | 67 --- .../tests/test_avro_encoder.py | 500 ------------------ .../tests/test_avro_encoder_async.py | 476 ----------------- 61 files changed, 6176 deletions(-) delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/CHANGELOG.md delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/LICENSE delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/MANIFEST.in delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/README.md delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_abstract_avro_encoder.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_apache_avro_encoder.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_constants.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_message_protocol.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_schema_registry_avro_encoder.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_version.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/__init__.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_async_lru.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_schema_registry_avro_encoder_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/exceptions.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/dev_requirements.txt delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/mypy.ini delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/README.md delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_event_data_message_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_with_metadata_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_with_callback_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_receive_integration_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_send_integration_async.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_event_data_message.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_with_metadata.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_with_callback.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_receive_integration.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_send_integration.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/sdk_packaging.toml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/setup.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_decode_readers_schema.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_with_auto_register_schemas.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_without_auto_register_schemas.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_encode_primitive.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_encode_record.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_parse_error_schema_as_record.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_parse_record_name.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_decode_readers_schema.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_with_auto_register_schemas.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_without_auto_register_schemas.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_encode_primitive.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_encode_record.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_parse_error_schema_as_record.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_parse_record_name.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_parse_record_name.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_serialize_record.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/test_avro_encoder.py delete mode 100644 sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/test_avro_encoder_async.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avroencoder/CHANGELOG.md deleted file mode 100644 index 8bb22791a80a..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/CHANGELOG.md +++ /dev/null @@ -1,17 +0,0 @@ -# Release History - -## 1.0.0b1 (2022-02-09) - -This version and all future versions will require Python 3.6+. Python 2.7 is no longer supported. - -### Features Added - -- This package is meant to replace the azure-schemaregistry-avroserializer. -- APIs have been updated to allow for encoding directly to and decoding from message type objects, where the data value is the Avro encoded payload. -- The content type of the message will hold the schema ID and record format indicator. - -### Other Changes - -- This beta release will be backward compatible for decoding data that was encoded with the AvroSerializer. -- The `encode` and `decode` methods on `AvroEncoder` support the following message models: - - `azure.eventhub.EventData` in `azure-eventhub==5.9.0b1` diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/LICENSE b/sdk/schemaregistry/azure-schemaregistry-avroencoder/LICENSE deleted file mode 100644 index 63447fd8bbbf..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) Microsoft Corporation. - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/MANIFEST.in b/sdk/schemaregistry/azure-schemaregistry-avroencoder/MANIFEST.in deleted file mode 100644 index a45dacf0a3f8..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -include *.md -include LICENSE -include azure/__init__.py -include azure/schemaregistry/__init__.py -recursive-include tests *.py -recursive-include samples *.py diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/README.md b/sdk/schemaregistry/azure-schemaregistry-avroencoder/README.md deleted file mode 100644 index ac70890316c5..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/README.md +++ /dev/null @@ -1,329 +0,0 @@ -# Azure Schema Registry Avro Encoder client library for Python - -Azure Schema Registry is a schema repository service hosted by Azure Event Hubs, providing schema storage, versioning, -and management. This package provides an Avro encoder capable of encoding and decoding payloads containing -Schema Registry schema identifiers and Avro-encoded data. - -[Source code][source_code] | [Package (PyPi)][pypi] | [API reference documentation][api_reference] | [Samples][sr_avro_samples] | [Changelog][change_log] - -## _Disclaimer_ - -_Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to https://github.com/Azure/azure-sdk-for-python/issues/20691_ - -## Getting started - -### Install the package - -Install the Azure Schema Registry Avro Encoder client library and Azure Identity client library for Python with [pip][pip]: - -```Bash -pip install azure-schemaregistry-avroencoder azure-identity -``` - -### Prerequisites: -To use this package, you must have: -* Azure subscription - [Create a free account][azure_sub] -* [Azure Schema Registry][schemaregistry_service] -* Python 3.6 or later - [Install Python][python] - -### Authenticate the client -Interaction with the Schema Registry Avro Encoder starts with an instance of AvroEncoder class, which takes the schema group name and the [Schema Registry Client][schemaregistry_client] class. The client constructor takes the Event Hubs fully qualified namespace and and Azure Active Directory credential: - -* The fully qualified namespace of the Schema Registry instance should follow the format: `.servicebus.windows.net`. - -* An AAD credential that implements the [TokenCredential][token_credential_interface] protocol should be passed to the constructor. There are implementations of the `TokenCredential` protocol available in the -[azure-identity package][pypi_azure_identity]. To use the credential types provided by `azure-identity`, please install the Azure Identity client library for Python with [pip][pip]: - -```Bash -pip install azure-identity -``` - -* Additionally, to use the async API, you must first install an async transport, such as [aiohttp](https://pypi.org/project/aiohttp/): - -```Bash -pip install aiohttp -``` - -**Create AvroEncoder using the azure-schemaregistry library:** - -```python -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.identity import DefaultAzureCredential - -credential = DefaultAzureCredential() -# Namespace should be similar to: '.servicebus.windows.net' -fully_qualified_namespace = '<< FULLY QUALIFIED NAMESPACE OF THE SCHEMA REGISTRY >>' -group_name = '<< GROUP NAME OF THE SCHEMA >>' -schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, credential) -encoder = AvroEncoder(client=schema_registry_client, group_name=group_name) -``` - -## Key concepts - -### AvroEncoder - -Provides API to encode to and decode from Avro Binary Encoding plus a -content type with schema ID. Uses [SchemaRegistryClient][schemaregistry_client] to get schema IDs from schema content or vice versa. - -### Supported message models - -Support has been added to certain Azure Messaging SDK model classes for interoperability with the `AvroEncoder`. These models are subtypes of the `MessageType` protocol defined under the `azure.schemaregistry.encoder.avroencoder` namespace. Currently, the supported model classes are: - -- `azure.eventhub.EventData` for `azure-eventhub==5.9.0b1` - -### Message format - -If a message type that follows the MessageType protocol is provided to the encoder, it will encode the corresponding data and content type properties as follows: - -- `data`: Avro payload (in general, format-specific payload) - - Avro Binary Encoding - - NOT Avro Object Container File, which includes the schema and defeats the - purpose of this encoder to move the schema out of the message payload and - into the schema registry. - -- `content type`: a string of the format `avro/binary+`, where: - - `avro/binary` is the format indicator - - `` is the hexadecimal representation of GUID, same format and byte order as the string from the Schema Registry service. - -If message type or callback function is not provided, and by default, the encoder will create the following dict: -`{"data": , "content_type": 'avro/binary+' }` - -## Examples - -The following sections provide several code snippets covering some of the most common Schema Registry tasks, including: - -- [Encoding](#encoding) -- [Decoding](#decoding) -- [Event Hubs Sending Integration](#event-hubs-sending-integration) -- [Event Hubs Receiving Integration](#event-hubs-receiving-integration) - -### Encoding - -Use `AvroEncoder.encode` method to encode dict data with the given Avro schema. -The method will use a schema previously registered to the Schema Registry service and keep the schema cached for future encoding usage. It is also possible to avoid pre-registering the schema to the service and automatically register with the `encode` method by instantiating the `AvroEncoder` with the keyword argument `auto_register_schemas=True`. - -```python -import os -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.identity import DefaultAzureCredential -from azure.eventhub import EventData - -token_credential = DefaultAzureCredential() -fully_qualified_namespace = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -group_name = "" -name = "example.avro.User" -format = "Avro" - -definition = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - -schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, token_credential) -schema_register_client.register(group_name, name, definition, format) -encoder = AvroEncoder(client=schema_registry_client, group_name=group_name) - -with encoder: - dict_data = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - event_data = encoder.encode(dict_data, schema=definition, message_type=EventData) - - # OR - - metadata_dict = encoder.encode(dict_data, schema=definition) - event_data = EventData.from_message_data(metadata_dict["data"], metadata_dict["content_type"]) -``` - -### Decoding - -Use `AvroEncoder.decode` method to decode the bytes value into dict data by either: - - Passing in a message object that is a subtype of the MessageType protocol. - - Passing in a dict with keys `data`(type bytes) and `content_type` (type string). -The method automatically retrieves the schema from the Schema Registry Service and keeps the schema cached for future decoding usage. - -```python -import os -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.identity import DefaultAzureCredential - -token_credential = DefaultAzureCredential() -fully_qualified_namespace = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -group_name = "" - -schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, token_credential) -encoder = AvroEncoder(client=schema_registry_client, group_name=group_name) - -with encoder: - # event_data is an EventData object with Avro encoded body - decoded_data = encoder.decode(event_data) - - # OR - - encoded_bytes = b'' - content_type = 'avro/binary+' - data_dict = {"data": encoded_bytes, "content_type": content_type} - decoded_data = encoder.decode(data_dict) -``` - -### Event Hubs Sending Integration - -Integration with [Event Hubs][eventhubs_repo] to send encoded Avro dict data as the body of EventData. - -```python -import os -from azure.eventhub import EventHubProducerClient, EventData -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.identity import DefaultAzureCredential - -token_credential = DefaultAzureCredential() -fully_qualified_namespace = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -group_name = "" -eventhub_connection_str = os.environ['EVENT_HUB_CONN_STR'] -eventhub_name = os.environ['EVENT_HUB_NAME'] - -definition = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - -schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, token_credential) -avro_encoder = AvroEncoder(client=schema_registry_client, group_name=group_name, auto_register_schemas=True) - -eventhub_producer = EventHubProducerClient.from_connection_string( - conn_str=eventhub_connection_str, - eventhub_name=eventhub_name -) - -with eventhub_producer, avro_encoder: - event_data_batch = eventhub_producer.create_batch() - dict_data = {"name": "Bob", "favorite_number": 7, "favorite_color": "red"} - event_data = avro_encoder.encode(dict_data, schema=definition, message_type=EventData) - event_data_batch.add(event_data) - eventhub_producer.send_batch(event_data_batch) -``` - -### Event Hubs Receiving Integration - -Integration with [Event Hubs][eventhubs_repo] to receive `EventData` and decoded raw bytes into Avro dict data. - -```python -import os -from azure.eventhub import EventHubConsumerClient -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.identity import DefaultAzureCredential - -token_credential = DefaultAzureCredential() -fully_qualified_namespace = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -group_name = "" -eventhub_connection_str = os.environ['EVENT_HUB_CONN_STR'] -eventhub_name = os.environ['EVENT_HUB_NAME'] - -schema_registry_client = SchemaRegistryClient(fully_qualified_namespace, token_credential) -avro_encoder = AvroEncoder(client=schema_registry_client, group_name=group_name) - -eventhub_consumer = EventHubConsumerClient.from_connection_string( - conn_str=eventhub_connection_str, - consumer_group='$Default', - eventhub_name=eventhub_name, -) - -def on_event(partition_context, event): - decoded_data = avro_encoder.decode(event) - -with eventhub_consumer, avro_encoder: - eventhub_consumer.receive(on_event=on_event, starting_position="-1") -``` - -## Troubleshooting - -### General - -Azure Schema Registry Avro Encoder raises exceptions defined in [Azure Core][azure_core]. - -### Logging -This library uses the standard -[logging][python_logging] library for logging. -Basic information about HTTP sessions (URLs, headers, etc.) is logged at INFO -level. - -Detailed DEBUG level logging, including request/response bodies and unredacted -headers, can be enabled on a client with the `logging_enable` argument: -```python -import sys -import logging -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.identity import DefaultAzureCredential - -# Create a logger for the SDK -logger = logging.getLogger('azure.schemaregistry') -logger.setLevel(logging.DEBUG) - -# Configure a console output -handler = logging.StreamHandler(stream=sys.stdout) -logger.addHandler(handler) - -credential = DefaultAzureCredential() -schema_registry_client = SchemaRegistryClient("", credential, logging_enable=True) -# This client will log detailed information about its HTTP sessions, at DEBUG level -encoder = AvroEncoder(client=schema_registry_client, group_name="") -``` - -Similarly, `logging_enable` can enable detailed logging for a single operation, -even when it isn't enabled for the client: -```py -encoder.encode(dict_data, schema=schema_definition, logging_enable=True) -``` - -## Next steps - -### More sample code - -Please find further examples in the [samples][sr_avro_samples] directory demonstrating common Azure Schema Registry Avro Encoder scenarios. - -## Contributing - -This project welcomes contributions and suggestions. Most contributions require you to agree to a -Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit https://cla.microsoft.com. - -When you submit a pull request, a CLA-bot will automatically determine whether you need to provide -a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions -provided by the bot. You will only need to do this once across all repos using our CLA. - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - - -[pip]: https://pypi.org/project/pip/ -[pypi]: https://pypi.org/project/azure-schemaregistry-avroencoder/ -[python]: https://www.python.org/downloads/ -[azure_core]: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/README.md -[azure_sub]: https://azure.microsoft.com/free/ -[python_logging]: https://docs.python.org/3/library/logging.html -[sr_avro_samples]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples -[api_reference]: https://docs.microsoft.com/python/api/overview/azure/schemaregistry-avroencoder-readme -[source_code]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder -[change_log]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder/CHANGELOG.md -[schemaregistry_client]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry -[schemaregistry_service]: https://aka.ms/schemaregistry -[eventhubs_repo]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/eventhub/azure-eventhub -[token_credential_interface]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/core/azure-core/azure/core/credentials.py -[pypi_azure_identity]: https://pypi.org/project/azure-identity/ \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/__init__.py deleted file mode 100644 index 80f86cb969ec..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/__init__.py deleted file mode 100644 index 80f86cb969ec..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/__init__.py deleted file mode 100644 index 80f86cb969ec..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/__init__.py deleted file mode 100644 index 82dce90e5ed6..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -from ._version import VERSION - -__version__ = VERSION - -from ._schema_registry_avro_encoder import AvroEncoder -from ._message_protocol import MessageType, MessageMetadataDict - -__all__ = [ - "AvroEncoder", - "MessageType", - "MessageMetadataDict" -] diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_abstract_avro_encoder.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_abstract_avro_encoder.py deleted file mode 100644 index 9fa10175147a..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_abstract_avro_encoder.py +++ /dev/null @@ -1,68 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from typing import BinaryIO, TypeVar, Union, Optional -from abc import abstractmethod - -ObjectType = TypeVar("ObjectType") - -class AbstractAvroObjectEncoder(object): - """ - An Avro encoder used for encoding/decoding an Avro RecordSchema. - """ - - @abstractmethod - def get_schema_fullname( - self, - schema, - ): - # type: (str) -> str - """ - Returns the namespace-qualified name of the provided schema. - Schema must be a Avro RecordSchema: - https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - :param schema: An Avro RecordSchema - :type schema: str - :rtype: str - """ - - - @abstractmethod - def encode( - self, - data, - schema, - ): - # type: (ObjectType, str) -> bytes - """Convert the provided value to it's binary representation and write it to the stream. - Schema must be a Avro RecordSchema: - https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - :param data: An object to encode - :type data: ObjectType - :param schema: An Avro RecordSchema - :type schema: str - :returns: Encoded bytes - :rtype: bytes - """ - - @abstractmethod - def decode( - self, - data: Union[bytes, BinaryIO], - schema: str, - *, - readers_schema: Optional[str] - ): - """Read the binary representation into a specific type. - Return type will be ignored, since the schema is deduced from the provided bytes. - :param data: A stream of bytes or bytes directly - :type data: BinaryIO or bytes - :param schema: An Avro RecordSchema - :type schema: str - :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. - :paramtype readers_schema: str or None - :returns: An instantiated object - :rtype: ObjectType - """ diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_apache_avro_encoder.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_apache_avro_encoder.py deleted file mode 100644 index b8a803c36344..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_apache_avro_encoder.py +++ /dev/null @@ -1,100 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from functools import lru_cache -from typing import BinaryIO, Union, TypeVar -from io import BytesIO -import avro -from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder - -from ._abstract_avro_encoder import AbstractAvroObjectEncoder - -ObjectType = TypeVar("ObjectType") - - -class ApacheAvroObjectEncoder(AbstractAvroObjectEncoder): - - def __init__(self, codec=None): - """A Avro encoder using avro lib from Apache. - :param str codec: The writer codec. If None, let the avro library decides. - """ - self._writer_codec = codec - - @lru_cache(maxsize=128) - def parse_schema(self, schema): # pylint: disable=no-self-use - return avro.schema.parse(schema) - - def get_schema_fullname(self, schema): - parsed_schema = self.parse_schema(schema) - return parsed_schema.fullname - - @lru_cache(maxsize=128) - def get_schema_writer(self, schema): # pylint: disable=no-self-use - schema = self.parse_schema(schema) - return DatumWriter(schema) - - @lru_cache(maxsize=128) - def get_schema_reader(self, schema, readers_schema): # pylint: disable=no-self-use - schema = self.parse_schema(schema) - if readers_schema: - readers_schema = self.parse_schema(readers_schema) - return DatumReader(writers_schema=schema, readers_schema=readers_schema) - - # pylint: disable=no-self-use - def encode( - self, - data, # type: ObjectType - schema, # type: Union[str, bytes, avro.schema.Schema] - ) -> bytes: - """Convert the provided value to it's binary representation and write it to the stream. - Schema must be a Avro RecordSchema: - https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - :param data: An object to encode - :type data: ObjectType - :param schema: An Avro RecordSchema - :type schema: str - :returns: Encoded bytes - :rtype: bytes - """ - if not schema: - raise ValueError("Schema is required in Avro encoder.") - - writer = self.get_schema_writer(schema) - - stream = BytesIO() - with stream: - writer.write(data, BinaryEncoder(stream)) - encoded_data = stream.getvalue() - return encoded_data - - # pylint: disable=no-self-use - def decode( - self, - data, # type: Union[bytes, BinaryIO] - schema, # type: Union[str, bytes, avro.schema.Schema] - *, - readers_schema=None, # type: Optional[Union[str, bytes, avro.schema.Schema]] - ) -> ObjectType: - """Read the binary representation into a specific type. - Return type will be ignored, since the schema is deduced from the provided bytes. - :param data: A stream of bytes or bytes directly - :type data: BinaryIO or bytes - :param schema: An Avro RecordSchema - :type schema: str - :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. - :paramtype readers_schema: str or None - :returns: An instantiated object - :rtype: ObjectType - """ - if not hasattr(data, 'read'): - data = BytesIO(data) - - reader = self.get_schema_reader(schema, readers_schema) - - with data: - bin_decoder = BinaryDecoder(data) - decoded_data = reader.read(bin_decoder) - - return decoded_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_constants.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_constants.py deleted file mode 100644 index 2deb93932e05..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_constants.py +++ /dev/null @@ -1,40 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- - -AVRO_MIME_TYPE = "avro/binary" - -# encoded payload used to consist of: -# 0~3: 4 bytes denoting record format identifier -# 4~35: 32 bytes denoting schema id -# 36~END: encode data - -# keeping temporarily for backward compatibility - -RECORD_FORMAT_IDENTIFIER_START_INDEX = 0 -RECORD_FORMAT_IDENTIFIER_LENGTH = 4 -SCHEMA_ID_START_INDEX = 4 -SCHEMA_ID_LENGTH = 32 -DATA_START_INDEX = 36 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_message_protocol.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_message_protocol.py deleted file mode 100644 index ee69f0c10fe7..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_message_protocol.py +++ /dev/null @@ -1,39 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any -try: - from typing import Protocol, TypedDict -except ImportError: - from typing_extensions import Protocol, TypedDict - - -class MessageMetadataDict(TypedDict): - """A dict with required keys: - - `data`: bytes - - `content_type`: str - """ - - data: bytes - content_type: str - -class MessageType(Protocol): - """Message Types that set and get data and content type values internally. - """ - - @classmethod - def from_message_data(cls, data: bytes, content_type: str, **kwargs: Any) -> "MessageType": - """ - Creates an object that is a subtype of MessageType given content type and - a data value to be set as body. - - :param bytes data: The data value to be set as the body of the message. - :param str content_type: The content type to be set on the message. - :rtype: ~azure.schemaregistry.encoder.avroencoder.MessageType - """ - ... - - def __message_data__(self) -> MessageMetadataDict: - ... diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_schema_registry_avro_encoder.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_schema_registry_avro_encoder.py deleted file mode 100644 index 2638607d4090..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_schema_registry_avro_encoder.py +++ /dev/null @@ -1,285 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -from functools import lru_cache -from io import BytesIO -from typing import Any, Dict, Mapping, Optional, Union, Callable - -from .exceptions import ( - SchemaParseError, - SchemaEncodeError, - SchemaDecodeError, -) -from ._apache_avro_encoder import ApacheAvroObjectEncoder as AvroObjectEncoder -from ._message_protocol import MessageMetadataDict, MessageType -from ._constants import ( - SCHEMA_ID_START_INDEX, - SCHEMA_ID_LENGTH, - DATA_START_INDEX, - AVRO_MIME_TYPE, - RECORD_FORMAT_IDENTIFIER_LENGTH, -) - - -class AvroEncoder(object): - """ - AvroEncoder provides the ability to encode and decode data according - to the given avro schema. It would automatically register, get and cache the schema. - - :keyword client: Required. The schema registry client - which is used to register schema and retrieve schema from the service. - :paramtype client: ~azure.schemaregistry.SchemaRegistryClient - :keyword str group_name: Required. Schema group under which schema should be registered. - :keyword bool auto_register_schemas: When true, register new schemas passed to encode. - Otherwise, and by default, encode will fail if the schema has not been pre-registered in the registry. - - """ - - def __init__(self, **kwargs): - # type: (Any) -> None - try: - self._schema_group = kwargs.pop("group_name") - self._schema_registry_client = kwargs.pop( - "client" - ) # type: "SchemaRegistryClient" - except KeyError as e: - raise TypeError("'{}' is a required keyword.".format(e.args[0])) - self._avro_encoder = AvroObjectEncoder(codec=kwargs.get("codec")) - self._auto_register_schemas = kwargs.get("auto_register_schemas", False) - self._auto_register_schema_func = ( - self._schema_registry_client.register_schema - if self._auto_register_schemas - else self._schema_registry_client.get_schema_properties - ) - - def __enter__(self): - # type: () -> AvroEncoder - self._schema_registry_client.__enter__() - return self - - def __exit__(self, *exc_details): - # type: (Any) -> None - self._schema_registry_client.__exit__(*exc_details) - - def close(self): - # type: () -> None - """This method is to close the sockets opened by the client. - It need not be used when using with a context manager. - """ - self._schema_registry_client.close() - - @lru_cache(maxsize=128) - def _get_schema_id(self, schema_name, schema_str, **kwargs): - # type: (str, str, Any) -> str - """ - Get schema id from local cache with the given schema. - If there is no item in the local cache, get schema id from the service and cache it. - - :param schema_name: Name of the schema - :type schema_name: str - :param str schema_str: Schema string - :return: Schema Id - :rtype: str - """ - schema_id = self._auto_register_schema_func( - self._schema_group, schema_name, schema_str, "Avro", **kwargs - ).id - return schema_id - - @lru_cache(maxsize=128) - def _get_schema(self, schema_id, **kwargs): - # type: (str, Any) -> str - """ - Get schema content from local cache with the given schema id. - If there is no item in the local cache, get schema from the service and cache it. - - :param str schema_id: Schema id - :return: Schema content - :rtype: str - """ - schema_str = self._schema_registry_client.get_schema( - schema_id, **kwargs - ).definition - return schema_str - - def encode( - self, - data: Mapping[str, Any], - *, - schema: str, - message_type: Optional[Callable] = None, - **kwargs: Any, - ) -> Union[MessageType, MessageMetadataDict]: - """ - Encode data with the given schema. Create content type value, which consists of the Avro Mime Type string - and the schema ID corresponding to given schema. If provided with a message constructor callback, - pass encoded data and content type to create message object. If not provided, return the following dict: - {"data": Avro encoded value, "content_type": Avro mime type string + schema ID}. - - If `message_type` is set, then additional keyword arguments will be passed to the message callback - function provided. - - Schema must be an Avro RecordSchema: - https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - - :param data: The data to be encoded. - :type data: Mapping[str, Any] - :keyword schema: Required. The schema used to encode the data. - :paramtype schema: str - :keyword message_type: The callback function or message class to construct the message. If message class, - it must be a subtype of the azure.schemaregistry.encoder.avroencoder.MessageType protocol. - If callback function, it must have the following method signature: - `(data: bytes, content_type: str, **kwargs) -> MessageType`, where `data` and `content_type` - are positional parameters. - :paramtype message_type: Callable or None - :rtype: MessageType or MessageMetadataDict - :raises ~azure.schemaregistry.encoder.avroencoder.exceptions.SchemaParseError: - Indicates an issue with parsing schema. - :raises ~azure.schemaregistry.encoder.avroencoder.exceptions.SchemaEncodeError: - Indicates an issue with encoding data for provided schema. - """ - - raw_input_schema = schema - - try: - schema_fullname = self._avro_encoder.get_schema_fullname(raw_input_schema) - except Exception as e: # pylint:disable=broad-except - SchemaParseError( - f"Cannot parse schema: {raw_input_schema}", error=e - ).raise_with_traceback() - - schema_id = self._get_schema_id(schema_fullname, raw_input_schema) - content_type = f"{AVRO_MIME_TYPE}+{schema_id}" - - try: - data_bytes = self._avro_encoder.encode(data, raw_input_schema) - except Exception as e: # pylint:disable=broad-except - SchemaEncodeError( - "Cannot encode value '{}' for schema: {}".format( - data, raw_input_schema - ), - error=e, - ).raise_with_traceback() - - stream = BytesIO() - - stream.write(data_bytes) - stream.flush() - - payload = stream.getvalue() - stream.close() - - if message_type: - try: - return message_type.from_message_data(payload, content_type, **kwargs) - except AttributeError: - try: - return message_type(payload, content_type, **kwargs) - except TypeError as e: - SchemaEncodeError( - f"""The data model {str(message_type)} is not a Callable that takes `data` - and `content_type` or a subtype of the MessageType protocol. - If using an Azure SDK model class, please check the README.md for the full list - of supported Azure SDK models and their corresponding versions.""" - ).raise_with_traceback() - - return {"data": payload, "content_type": content_type} - - def _convert_preamble_format( - self, data, content_type - ): # pylint: disable=no-self-use - record_format_identifier = b"\0\0\0\0" - if data[0:RECORD_FORMAT_IDENTIFIER_LENGTH] == record_format_identifier: - schema_id = data[ - SCHEMA_ID_START_INDEX : (SCHEMA_ID_START_INDEX + SCHEMA_ID_LENGTH) - ].decode("utf-8") - content_type = f"{AVRO_MIME_TYPE}+{schema_id}" - data = data[DATA_START_INDEX:] - - return data, content_type - - def decode( - self, - message: Union[MessageType, MessageMetadataDict], - *, - readers_schema: Optional[str] = None, - **kwargs, # pylint: disable=unused-argument - ) -> Dict[str, Any]: - """ - Decode bytes data using schema ID in the content type field. `message` must be one of the following: - 1) A Subtype of the MessageType protocol. - 2) A dict {"data": ..., "content_type": ...}, where "data" is bytes and "content_type" is string. - 3) If using to decode data that was serialized with the AvroSerializer, a dict - {"data": ..., "content_type": None}, where "data" is bytes and "content_type" is None. - Data must follow format of associated Avro RecordSchema: - https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - - :param message: The message object which holds the data to be decoded and content type - containing the schema ID. - :type message: MessageType or MessageMetadataDict - :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. - :paramtype readers_schema: str or None - :rtype: Dict[str, Any] - :raises ~azure.schemaregistry.encoder.avroencoder.exceptions.SchemaParseError: - Indicates an issue with parsing schema. - :raises ~azure.schemaregistry.encoder.avroencoder.exceptions.SchemaDecodeError: - Indicates an issue with decoding value. - """ - - try: - message_data_dict = message.__message_data__() - data = message_data_dict["data"] - content_type = message_data_dict["content_type"] - except AttributeError: - try: - data = message["data"] - content_type = message["content_type"] - except (KeyError, TypeError): - SchemaDecodeError( - f"""The data model {str(message)} is not a subtype of the MessageType protocol or type - MessageMetadataDict. If using an Azure SDK model class, please check the README.md - for the full list of supported Azure SDK models and their corresponding versions.""" - ).raise_with_traceback() - - # include in first preview for back compatibility - data, content_type = self._convert_preamble_format(data, content_type) - - schema_id = content_type.split("+")[1] - schema_definition = self._get_schema(schema_id) - try: - dict_value = self._avro_encoder.decode( - data, schema_definition, readers_schema=readers_schema - ) - except Exception as e: # pylint:disable=broad-except - error_message = ( - f"Cannot decode value '{data}' for schema: {schema_definition}\n and reader's schema: {readers_schema}" - if readers_schema - else f"Cannot decode value '{data}' for schema: {schema_definition}" - ) - SchemaDecodeError( - error_message, - error=e, - ).raise_with_traceback() - return dict_value diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_version.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_version.py deleted file mode 100644 index d316addb36cb..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_version.py +++ /dev/null @@ -1,27 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- - -VERSION = "1.0.0b1" diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/__init__.py deleted file mode 100644 index dfee0c285ae1..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -from ._schema_registry_avro_encoder_async import AvroEncoder - -__all__ = [ - "AvroEncoder" -] diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_async_lru.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_async_lru.py deleted file mode 100644 index c51dfdf0c9dd..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_async_lru.py +++ /dev/null @@ -1,230 +0,0 @@ -# -------------------------------------------------------------------------- -# The MIT License -# -# Copyright (c) 2018 aio-libs team https://github.com/aio-libs/ -# Copyright (c) 2017 Ocean S. A. https://ocean.io/ -# Copyright (c) 2016-2017 WikiBusiness Corporation http://wikibusiness.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -------------------------------------------------------------------------- -# Copying over `async_lru.py`[https://github.com/aio-libs/async-lru/blob/master/async_lru.py] -# from `aio-libs`[https://github.com/aio-libs/async-lru] for the following reasons: -# 1. There has not been an official release of `async_lru` in 2 years. -# 2. The last update to the library was a year ago, so it seems the library is -# not being actively maintained. - -import asyncio -from collections import OrderedDict -from functools import _CacheInfo, _make_key, partial, wraps - - -__version__ = "1.0.2" - -__all__ = ("alru_cache",) - - -def unpartial(fn): - while hasattr(fn, "func"): - fn = fn.func - - return fn - - -def _done_callback(fut, task): - if task.cancelled(): - fut.cancel() - return - - exc = task.exception() - if exc is not None: - fut.set_exception(exc) - return - - fut.set_result(task.result()) - - -def _cache_invalidate(wrapped, typed, *args, **kwargs): - # pylint: disable=protected-access - key = _make_key(args, kwargs, typed) - - exists = key in wrapped._cache - - if exists: - wrapped._cache.pop(key) - - return exists - - -def _cache_clear(wrapped): - # pylint: disable=protected-access - wrapped.hits = wrapped.misses = 0 - wrapped._cache = OrderedDict() - wrapped.tasks = set() - - -def _open(wrapped): - if not wrapped.closed: - raise RuntimeError("alru_cache is not closed") - - # pylint: disable=protected-access - was_closed = ( - wrapped.hits == wrapped.misses == len(wrapped.tasks) == len(wrapped._cache) == 0 - ) - - if not was_closed: - raise RuntimeError("alru_cache was not closed correctly") - - wrapped.closed = False - - -def _close(wrapped, *, cancel=False, return_exceptions=True): - if wrapped.closed: - raise RuntimeError("alru_cache is closed") - - wrapped.closed = True - - if cancel: - for task in wrapped.tasks: - if not task.done(): # not sure is it possible - task.cancel() - - return _wait_closed(wrapped, return_exceptions=return_exceptions) - - -async def _wait_closed(wrapped, *, return_exceptions): - wait_closed = asyncio.gather(*wrapped.tasks, return_exceptions=return_exceptions) - - wait_closed.add_done_callback(partial(_close_waited, wrapped)) - - ret = await wait_closed - - # hack to get _close_waited callback to be executed - await asyncio.sleep(0) - - return ret - - -def _close_waited(wrapped, _): - wrapped.cache_clear() - - -def _cache_info(wrapped, maxsize): - # pylint: disable=protected-access - return _CacheInfo( - wrapped.hits, - wrapped.misses, - maxsize, - len(wrapped._cache), - ) - - -def __cache_touch(wrapped, key): - # pylint: disable=protected-access - try: - wrapped._cache.move_to_end(key) - except KeyError: # not sure is it possible - pass - - -def _cache_hit(wrapped, key): - wrapped.hits += 1 - __cache_touch(wrapped, key) - - -def _cache_miss(wrapped, key): - wrapped.misses += 1 - __cache_touch(wrapped, key) - - -def alru_cache( - fn=None, - maxsize=128, - typed=False, - *, - cache_exceptions=True, -): - def wrapper(fn): - # pylint: disable=protected-access - _origin = unpartial(fn) - - if not asyncio.iscoroutinefunction(_origin): - raise RuntimeError("Coroutine function is required, got {}".format(fn)) - - # functools.partialmethod support - if hasattr(fn, "_make_unbound_method"): - fn = fn._make_unbound_method() - - @wraps(fn) - async def wrapped(*fn_args, **fn_kwargs): - if wrapped.closed: - raise RuntimeError("alru_cache is closed for {}".format(wrapped)) - - loop = asyncio.get_event_loop() - - key = _make_key(fn_args, fn_kwargs, typed) - - fut = wrapped._cache.get(key) - - if fut is not None: - if not fut.done(): - _cache_hit(wrapped, key) - return await asyncio.shield(fut) - - exc = fut._exception - - if exc is None or cache_exceptions: - _cache_hit(wrapped, key) - return fut.result() - - # exception here and cache_exceptions == False - wrapped._cache.pop(key) - - fut = loop.create_future() - task = loop.create_task(fn(*fn_args, **fn_kwargs)) - task.add_done_callback(partial(_done_callback, fut)) - - wrapped.tasks.add(task) - task.add_done_callback(wrapped.tasks.remove) - - wrapped._cache[key] = fut - - if maxsize is not None and len(wrapped._cache) > maxsize: - wrapped._cache.popitem(last=False) - - _cache_miss(wrapped, key) - return await asyncio.shield(fut) - - _cache_clear(wrapped) - wrapped._origin = _origin - wrapped.closed = False - wrapped.cache_info = partial(_cache_info, wrapped, maxsize) - wrapped.cache_clear = partial(_cache_clear, wrapped) - wrapped.invalidate = partial(_cache_invalidate, wrapped, typed) - wrapped.close = partial(_close, wrapped) - wrapped.open = partial(_open, wrapped) - - return wrapped - - if fn is None: - return wrapper - - if callable(fn) or hasattr(fn, "_make_unbound_method"): - return wrapper(fn) - - raise NotImplementedError("{} decorating is not supported".format(fn)) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_schema_registry_avro_encoder_async.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_schema_registry_avro_encoder_async.py deleted file mode 100644 index c3ae535ff289..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_schema_registry_avro_encoder_async.py +++ /dev/null @@ -1,277 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -from io import BytesIO -from typing import Any, Callable, Dict, Mapping, Union, Optional -from ._async_lru import alru_cache -from .._constants import ( - SCHEMA_ID_START_INDEX, - SCHEMA_ID_LENGTH, - DATA_START_INDEX, - AVRO_MIME_TYPE, - RECORD_FORMAT_IDENTIFIER_LENGTH, -) -from .._message_protocol import MessageType, MessageMetadataDict -from .._apache_avro_encoder import ApacheAvroObjectEncoder as AvroObjectEncoder -from ..exceptions import ( - SchemaParseError, - SchemaEncodeError, - SchemaDecodeError, -) - - -class AvroEncoder(object): - """ - AvroEncoder provides the ability to encode and decode data according - to the given avro schema. It would automatically register, get and cache the schema. - - :keyword client: Required. The schema registry client - which is used to register schema and retrieve schema from the service. - :paramtype client: ~azure.schemaregistry.aio.SchemaRegistryClient - :keyword str group_name: Required. Schema group under which schema should be registered. - :keyword bool auto_register_schemas: When true, register new schemas passed to encode. - Otherwise, and by default, encode will fail if the schema has not been pre-registered in the registry. - - """ - - def __init__(self, **kwargs): - # type: (Any) -> None - try: - self._schema_group = kwargs.pop("group_name") - self._schema_registry_client = kwargs.pop( - "client" - ) # type: "SchemaRegistryClient" - except KeyError as e: - raise TypeError("'{}' is a required keyword.".format(e.args[0])) - self._avro_encoder = AvroObjectEncoder(codec=kwargs.get("codec")) - self._auto_register_schemas = kwargs.get("auto_register_schemas", False) - self._auto_register_schema_func = ( - self._schema_registry_client.register_schema - if self._auto_register_schemas - else self._schema_registry_client.get_schema_properties - ) - - async def __aenter__(self): - # type: () -> AvroEncoder - await self._schema_registry_client.__aenter__() - return self - - async def __aexit__(self, *exc_details): - # type: (Any) -> None - await self._schema_registry_client.__aexit__(*exc_details) - - async def close(self): - # type: () -> None - """This method is to close the sockets opened by the client. - It need not be used when using with a context manager. - """ - await self._schema_registry_client.close() - - @alru_cache(maxsize=128, cache_exceptions=False) - async def _get_schema_id(self, schema_name, schema_str, **kwargs): - # type: (str, str, Any) -> str - """ - Get schema id from local cache with the given schema. - If there is no item in the local cache, get schema id from the service and cache it. - - :param schema_name: Name of the schema - :type schema_name: str - :param str schema_str: Schema string - :return: Schema Id - :rtype: str - """ - schema_properties = await self._auto_register_schema_func( - self._schema_group, schema_name, schema_str, "Avro", **kwargs - ) - return schema_properties.id - - @alru_cache(maxsize=128, cache_exceptions=False) - async def _get_schema(self, schema_id, **kwargs): - # type: (str, Any) -> str - """ - Get schema definition from local cache with the given schema id. - If there is no item in the local cache, get schema from the service and cache it. - - :param str schema_id: Schema id - :return: Schema definition - """ - schema = await self._schema_registry_client.get_schema(schema_id, **kwargs) - return schema.definition - - async def encode( - self, - data: Mapping[str, Any], - *, - schema: str, - message_type: Optional[Callable] = None, - **kwargs: Any, - ) -> Union[MessageType, MessageMetadataDict]: - - """ - Encode data with the given schema. Create content type value, which consists of the Avro Mime Type string - and the schema ID corresponding to given schema. If provided with a message constructor callback, - pass encoded data and content type to create message object. If not provided, return the following dict: - {"data": Avro encoded value, "content_type": Avro mime type string + schema ID}. - - If `message_type` is set, then additional keyword arguments will be passed to the message callback - function provided. - - Schema must be an Avro RecordSchema: - https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - - :param data: The data to be encoded. - :type data: Mapping[str, Any] - :keyword schema: Required. The schema used to encode the data. - :paramtype schema: str - :keyword message_type: The callback function or message class to construct the message. If message class, - it must be a subtype of the azure.schemaregistry.encoder.avroencoder.MessageType protocol. - If callback function, it must have the following method signature: - `(data: bytes, content_type: str, **kwargs) -> MessageType`, where `data` and `content_type` - are positional parameters. - :paramtype message_type: Callable or None - :rtype: MessageType or MessageMetadataDict - :raises ~azure.schemaregistry.encoder.avroencoder.exceptions.SchemaParseError: - Indicates an issue with parsing schema. - :raises ~azure.schemaregistry.encoder.avroencoder.exceptions.SchemaEncodeError: - Indicates an issue with encoding data for provided schema. - """ - - raw_input_schema = schema - - try: - schema_fullname = self._avro_encoder.get_schema_fullname(raw_input_schema) - except Exception as e: # pylint:disable=broad-except - SchemaParseError( - f"Cannot parse schema: {raw_input_schema}", error=e - ).raise_with_traceback() - - schema_id = await self._get_schema_id(schema_fullname, raw_input_schema) - content_type = f"{AVRO_MIME_TYPE}+{schema_id}" - - try: - data_bytes = self._avro_encoder.encode(data, raw_input_schema) - except Exception as e: # pylint:disable=broad-except - SchemaEncodeError( - "Cannot encode value '{}' for schema: {}".format( - data, raw_input_schema - ), - error=e, - ).raise_with_traceback() - - stream = BytesIO() - - stream.write(data_bytes) - stream.flush() - - payload = stream.getvalue() - stream.close() - if message_type: - try: - return message_type.from_message_data(payload, content_type, **kwargs) - except AttributeError: - try: - return message_type(payload, content_type, **kwargs) - except TypeError as e: - SchemaEncodeError( - f"""The data model {str(message_type)} is not a Callable that takes `data` - and `content_type` or a subtype of the MessageType protocol. - If using an Azure SDK model class, please check the README.md for the full list - of supported Azure SDK models and their corresponding versions.""" - ).raise_with_traceback() - - return {"data": payload, "content_type": content_type} - - def _convert_preamble_format(self, data, content_type): # pylint: disable=no-self-use - record_format_identifier = b"\0\0\0\0" - if data[0:RECORD_FORMAT_IDENTIFIER_LENGTH] == record_format_identifier: - schema_id = data[ - SCHEMA_ID_START_INDEX : (SCHEMA_ID_START_INDEX + SCHEMA_ID_LENGTH) - ].decode("utf-8") - content_type = f"{AVRO_MIME_TYPE}+{schema_id}" - data = data[DATA_START_INDEX:] - - return data, content_type - - async def decode( - self, - message: Union[MessageType, MessageMetadataDict], - *, - readers_schema: Optional[str] = None, - **kwargs, # pylint: disable=unused-argument - ) -> Dict[str, Any]: - """ - Decode bytes data using schema ID in the content type field. `message` must be one of the following: - 1) A Subtype of the MessageType protocol. - 2) A dict {"data": ..., "content_type": ...}, where "data" is bytes and "content_type" is string. - 3) If using to decode data that was serialized with the AvroSerializer, a dict - {"data": ..., "content_type": None}, where "data" is bytes and "content_type" is None. - Data must follow format of associated Avro RecordSchema: - https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - - :param message: The message object which holds the data to be decoded and content type - containing the schema ID. - :type message: MessageType or MessageMetadataDict - :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. - :paramtype readers_schema: str or None - :rtype: Dict[str, Any] - :raises ~azure.schemaregistry.encoder.avroencoder.exceptions.SchemaParseError: - Indicates an issue with parsing schema. - :raises ~azure.schemaregistry.encoder.avroencoder.exceptions.SchemaDecodeError: - Indicates an issue with decoding value. - """ - - try: - message_data_dict = message.__message_data__() - data = message_data_dict["data"] - content_type = message_data_dict["content_type"] - except AttributeError: - try: - data = message["data"] - content_type = message["content_type"] - except (KeyError, TypeError): - SchemaDecodeError( - f"""The data model {str(message)} is not a subtype of the MessageType protocol or type - MessageMetadataDict. If using an Azure SDK model class, please check the README.md - for the full list of supported Azure SDK models and their corresponding versions.""" - ).raise_with_traceback() - - # include in first preview for back compatibility - data, content_type = self._convert_preamble_format(data, content_type) - - schema_id = content_type.split("+")[1] - schema_definition = await self._get_schema(schema_id) - try: - dict_value = self._avro_encoder.decode(data, schema_definition, readers_schema=readers_schema) - except Exception as e: # pylint:disable=broad-except - error_message = ( - f"Cannot decode value '{data}' for schema: {schema_definition}\n and reader's schema: {readers_schema}" - if readers_schema - else f"Cannot decode value '{data}' for schema: {schema_definition}" - ) - SchemaDecodeError( - error_message, - error=e, - ).raise_with_traceback() - return dict_value diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/exceptions.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/exceptions.py deleted file mode 100644 index 66a5f730f0eb..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/exceptions.py +++ /dev/null @@ -1,69 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -from azure.core.exceptions import AzureError - -class SchemaParseError(AzureError): - """Error parsing a JSON schema. - :param str message: The message object stringified as 'message' attribute - :keyword error: The original exception, if any - - :ivar str message: A stringified version of the message parameter - :ivar inner_exception: The exception passed with the 'error' kwarg - :vartype inner_exception: Exception - :ivar exc_type: The exc_type from sys.exc_info() - :ivar exc_value: The exc_value from sys.exc_info() - :ivar exc_traceback: The exc_traceback from sys.exc_info() - :ivar exc_msg: A string formatting of message parameter, exc_type and exc_value - """ - - -class SchemaEncodeError(AzureError): - """Error encoding a JSON schema. - :param str message: The message object stringified as 'message' attribute - :keyword error: The original exception, if any - - :ivar str message: A stringified version of the message parameter - :ivar inner_exception: The exception passed with the 'error' kwarg - :vartype inner_exception: Exception - :ivar exc_type: The exc_type from sys.exc_info() - :ivar exc_value: The exc_value from sys.exc_info() - :ivar exc_traceback: The exc_traceback from sys.exc_info() - :ivar exc_msg: A string formatting of message parameter, exc_type and exc_value - """ - -class SchemaDecodeError(AzureError): - """Error decoding a JSON schema. - :param str message: The message object stringified as 'message' attribute - :keyword error: The original exception, if any - - :ivar str message: A stringified version of the message parameter - :ivar inner_exception: The exception passed with the 'error' kwarg - :vartype inner_exception: Exception - :ivar exc_type: The exc_type from sys.exc_info() - :ivar exc_value: The exc_value from sys.exc_info() - :ivar exc_traceback: The exc_traceback from sys.exc_info() - :ivar exc_msg: A string formatting of message parameter, exc_type and exc_value - """ diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/dev_requirements.txt b/sdk/schemaregistry/azure-schemaregistry-avroencoder/dev_requirements.txt deleted file mode 100644 index 14549f3dbe61..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/dev_requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ --e ../../../tools/azure-devtools --e ../../../tools/azure-sdk-tools --e ../../identity/azure-identity -aiohttp>=3.0 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/mypy.ini b/sdk/schemaregistry/azure-schemaregistry-avroencoder/mypy.ini deleted file mode 100644 index c5adef3f8a6d..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/mypy.ini +++ /dev/null @@ -1,6 +0,0 @@ -[mypy] -python_version = 3.7 -warn_unused_configs = True -ignore_missing_imports = True - -# Per-module options: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/README.md b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/README.md deleted file mode 100644 index 7e26100420da..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/README.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -page_type: sample -languages: - - python -products: - - azure - - azure-event-hubs -urlFragment: schemaregistry-avroencoder-samples ---- - -# Azure Schema Registry Avro Encoder library for Python Samples - -These are code samples that show common scenario operations with the Schema Registry Avro Encoder library. - -Several Schema Registry Avro Encoder Python SDK samples are available to you in the SDK's GitHub repository. These samples provide example code for additional scenarios commonly encountered while working with Schema Registry Avro Encoder: - -* [encode_and_decode_event_data_message.py][encode_and_decode_event_data_message_sample] ([async version][encode_and_decode_event_data_message_async_sample]) - Examples for common Schema Registry Avro Encoder tasks: - * Encode data according to the given schema and create EventData object - * Decode data given an EventData object with encoded data and corresponding content type -* [eventhub_send_integration.py][eventhub_send_integration_sample] ([async version][eventhub_send_integration_async_sample]) - Examples for integration with EventHub in sending tasks: - * Encode data with the given schema and send `EventData` to Event Hubs. -* [eventhub_receive_integration.py][eventhub_receive_integration_sample] ([async version][eventhub_receive_integration_async_sample]) - Examples for integration with EventHub in receiving tasks: - * Receive `EventData` from Event Hubs and decode the received bytes. - -## Prerequisites -- Python 3.6 or later. -- **Microsoft Azure Subscription:** To use Azure services, including Azure Schema Registry, you'll need a subscription. -If you do not have an existing Azure account, you may sign up for a free trial or use your MSDN subscriber benefits when you [create an account](https://account.windowsazure.com/Home/Index). - -## Setup - -1. Install the Azure Schema Registry Avro Encoder client library and Azure Identity client library for Python with [pip](https://pypi.org/project/pip/): - -```bash -pip install azure-schemaregistry-avroencoder azure-identity -``` - -Additionally, if using with `azure.eventhub.EventData`, install `azure-eventhub==5.9.0b1`: - -```bash -pip install azure-eventhub==5.9.0b1 -``` - -2. Clone or download this sample repository -3. Open the sample folder in Visual Studio Code or your IDE of choice. - -## Running the samples - -1. Open a terminal window and `cd` to the directory that the samples are saved in. -2. Set the environment variables specified in the sample file you wish to run. -3. Follow the usage described in the file, e.g. `python encode_and_decode_event_data_message.py` - -## Next steps - -Check out the [API reference documentation][api_reference] to learn more about -what you can do with the Azure Schema Registry Avro Encoder library. - - -[encode_and_decode_event_data_message_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_event_data_message.py -[eventhub_send_integration_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_send_integration.py -[eventhub_receive_integration_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_receive_integration.py -[encode_and_decode_event_data_message_async_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_event_data_message_async.py -[eventhub_send_integration_async_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_send_integration_async.py -[eventhub_receive_integration_async_sample]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_receive_integration_async.py -[api_reference]: https://docs.microsoft.com/python/api/overview/azure/schemaregistry-avroencoder-readme diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_event_data_message_async.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_event_data_message_async.py deleted file mode 100644 index df888dac06c3..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_event_data_message_async.py +++ /dev/null @@ -1,107 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import os -import asyncio - -from azure.identity.aio import ClientSecretCredential -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder.aio import AvroEncoder -from azure.eventhub import EventData - -TENANT_ID=os.environ['AZURE_TENANT_ID'] -CLIENT_ID=os.environ['AZURE_CLIENT_ID'] -CLIENT_SECRET=os.environ['AZURE_CLIENT_SECRET'] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE=os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -GROUP_NAME=os.environ['SCHEMAREGISTRY_GROUP'] -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET -) - - -async def encode_to_event_data_message(encoder): - dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - dict_data_alice = {"name": "Alice", "favorite_number": 15, "favorite_color": "green"} - - # Schema would be automatically registered into Schema Registry and cached locally. - event_data_ben = await encoder.encode( - dict_data_ben, schema=SCHEMA_STRING, message_type=EventData - ) - - # The second call won't trigger a service call. - event_data_alice = await encoder.encode( - dict_data_alice, schema=SCHEMA_STRING, message_type=EventData - ) - - print("Encoded data is: ", next(event_data_ben.body)) - print("Encoded data is: ", next(event_data_alice.body)) - - print("Encoded content type is: ", event_data_ben.content_type) - print("Encoded content type is: ", event_data_alice.content_type) - return [event_data_ben, event_data_alice] - - -async def decode_event_data_message(encoder, event_data): - # encoder.decode would extract the schema id from the content_type, - # retrieve schema from Schema Registry and cache the schema locally. - # If the schema id is in the local cache, the call won't trigger a service call. - decoded_data = await encoder.decode(event_data) - - print("Decoded data is: ", decoded_data) - return decoded_data - - -async def main(): - schema_registry = SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=token_credential, - ) - encoder = AvroEncoder( - client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True - ) - event_data_ben, event_data_alice = await encode_to_event_data_message(encoder) - decoded_data_ben = await decode_event_data_message(encoder, event_data_ben) - decoded_data_alice = await decode_event_data_message(encoder, event_data_alice) - await encoder.close() - await token_credential.close() - -if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_with_metadata_async.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_with_metadata_async.py deleted file mode 100644 index 2bb25c613f0a..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_with_metadata_async.py +++ /dev/null @@ -1,97 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import os -import asyncio - -from azure.identity.aio import ClientSecretCredential -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder.aio import AvroEncoder -from azure.eventhub import EventData - -TENANT_ID = os.environ["AZURE_TENANT_ID"] -CLIENT_ID = os.environ["AZURE_CLIENT_ID"] -CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ - "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" -] -GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET -) - - -async def encode_metadata_dict(encoder): - dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - encoded_metadata_ben = await encoder.encode(dict_data_ben, schema=SCHEMA_STRING) - - print("Encoded metadata is: ", encoded_metadata_ben) - return EventData.from_message_data( - encoded_metadata_ben["data"], - encoded_metadata_ben["content_type"], - ) - -async def decode_with_data_and_content_type(encoder, event_data): - # get data as bytes - data = bytearray() - for d in event_data.body: - data += d - data_bytes = bytes(data) - data_dict = {"data": data_bytes, "content_type": event_data.content_type} - decoded_data = await encoder.decode(data_dict) - - print("Decoded data is: ", decoded_data) - return decoded_data - - -async def main(): - schema_registry = SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=token_credential, - ) - encoder = AvroEncoder( - client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True - ) - event_data = await encode_metadata_dict(encoder) - decoded_data = await decode_with_data_and_content_type(encoder, event_data) - await encoder.close() - await token_credential.close() - -if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_with_callback_async.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_with_callback_async.py deleted file mode 100644 index c61759b8f561..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_with_callback_async.py +++ /dev/null @@ -1,88 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import os -import asyncio - -from azure.identity.aio import ClientSecretCredential -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder.aio import AvroEncoder -from azure.eventhub import EventData - -TENANT_ID = os.environ["AZURE_TENANT_ID"] -CLIENT_ID = os.environ["AZURE_CLIENT_ID"] -CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ - "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" -] -GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET -) - -async def encode_with_callback(encoder): - - # Callback MUST have these parameters in the following order: (data, content_type, **kwargs) - def sample_create_event_data(data, content_type, **kwargs): - print("Creating sample EventData with callback.") - return EventData.from_message_data(data, content_type, **kwargs) - - dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - event_data_ben = await encoder.encode( - dict_data_ben, schema=SCHEMA_STRING, message_type=sample_create_event_data - ) - - print("Encoded data is: ", next(event_data_ben.body)) - print("Encoded content type is: ", event_data_ben.content_type) - return event_data_ben - -async def main(): - schema_registry = SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=token_credential, - ) - encoder = AvroEncoder( - client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True - ) - event_data_ben = await encode_with_callback(encoder) - await encoder.close() - await token_credential.close() - -if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_receive_integration_async.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_receive_integration_async.py deleted file mode 100644 index 5667773e85eb..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_receive_integration_async.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -""" -Examples to show receiving events from EventHub with AvroEncoder integrated for data decoding. -""" - -# pylint: disable=C0111 -import os -import asyncio -from azure.eventhub.aio import EventHubConsumerClient -from azure.identity.aio import DefaultAzureCredential -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder.aio import AvroEncoder - -EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] -EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] - - -# create an EventHubConsumerClient instance -eventhub_consumer = EventHubConsumerClient.from_connection_string( - conn_str=EVENTHUB_CONNECTION_STR, - consumer_group='$Default', - eventhub_name=EVENTHUB_NAME, -) -# create a AvroEncoder instance -azure_credential = DefaultAzureCredential() -avro_encoder = AvroEncoder( - client=SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=azure_credential - ), - group_name=GROUP_NAME, - auto_register_schemas=True -) - -async def on_event(partition_context, event): - print(f"Received event from partition: {partition_context.partition_id}.") - - bytes_payload = b"".join(b for b in event.body) - print(f'The received bytes of the EventData is {bytes_payload}.') - - # Use the decode method to decode the payload of the event. - # The decode method will extract the schema id from the content_type, and automatically retrieve the Avro Schema - # from the Schema Registry Service. The schema will be cached locally for future usage. - decoded_data = await avro_encoder.decode(event) - print(f'The dict data after decoding is {decoded_data}') - - -async def main(): - try: - async with eventhub_consumer, avro_encoder: - await eventhub_consumer.receive( - on_event=on_event, - starting_position="-1", # "-1" is from the beginning of the partition. - ) - except KeyboardInterrupt: - print('Stopped receiving.') - finally: - await avro_encoder.close() - await azure_credential.close() - await eventhub_consumer.close() - -if __name__ == '__main__': - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_send_integration_async.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_send_integration_async.py deleted file mode 100644 index ed845f437062..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_send_integration_async.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -""" -Examples to show sending event to EventHub with AvroEncoder integrated for data serialization. -""" - -# pylint: disable=C0111 - -import os -import asyncio -from azure.eventhub import EventData -from azure.eventhub.aio import EventHubProducerClient -from azure.identity.aio import DefaultAzureCredential -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder.aio import AvroEncoder - -EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] -EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] - -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - -# create an EventHubProducerClient instance -eventhub_producer = EventHubProducerClient.from_connection_string( - conn_str=EVENTHUB_CONNECTION_STR, - eventhub_name=EVENTHUB_NAME -) -# create a AvroEncoder instance -azure_credential = DefaultAzureCredential() -# create a AvroEncoder instance -avro_encoder = AvroEncoder( - client=SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=azure_credential - ), - group_name=GROUP_NAME, - auto_register_schemas=True -) - -async def send_event_data_batch(producer, encoder): - event_data_batch = await producer.create_batch() - dict_data = {"name": "Bob", "favorite_number": 7, "favorite_color": "red"} - # Use the encode method to convert dict object to bytes with the given avro schema and set body of EventData. - # The encode method will automatically register the schema into the Schema Registry Service and - # schema will be cached locally for future usage. - event_data = await encoder.encode(data=dict_data, schema=SCHEMA_STRING, message_type=EventData) - print(f'The bytes of encoded dict data is {next(event_data.body)}.') - - event_data_batch.add(event_data) - await producer.send_batch(event_data_batch) - print('Send is done.') - - -async def main(): - - await send_event_data_batch(eventhub_producer, avro_encoder) - await avro_encoder.close() - await azure_credential.close() - await eventhub_producer.close() - -if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_event_data_message.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_event_data_message.py deleted file mode 100644 index ff8eb380cdd7..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_event_data_message.py +++ /dev/null @@ -1,101 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import os - -from azure.identity import ClientSecretCredential -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.eventhub import EventData - -TENANT_ID = os.environ["AZURE_TENANT_ID"] -CLIENT_ID = os.environ["AZURE_CLIENT_ID"] -CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ - "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" -] -GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET -) - - -def encode_to_event_data_message(encoder): - dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - dict_data_alice = {"name": "Alice", "favorite_number": 15, "favorite_color": "green"} - - # Schema would be automatically registered into Schema Registry and cached locally. - event_data_ben = encoder.encode( - dict_data_ben, schema=SCHEMA_STRING, message_type=EventData - ) - - # The second call won't trigger a service call. - event_data_alice = encoder.encode( - dict_data_alice, schema=SCHEMA_STRING, message_type=EventData - ) - - print("Encoded data is: ", next(event_data_ben.body)) - print("Encoded data is: ", next(event_data_alice.body)) - - print("Encoded content type is: ", event_data_ben.content_type) - print("Encoded content type is: ", event_data_alice.content_type) - return [event_data_ben, event_data_alice] - - -def decode_event_data_message(encoder, event_data): - # encoder.decode would extract the schema id from the content_type, - # retrieve schema from Schema Registry and cache the schema locally. - # If the schema id is in the local cache, the call won't trigger a service call. - decoded_data = encoder.decode(event_data) - - print("Decoded data is: ", decoded_data) - return decoded_data - - -if __name__ == "__main__": - schema_registry = SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=token_credential, - ) - encoder = AvroEncoder( - client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True - ) - event_data_ben, event_data_alice = encode_to_event_data_message(encoder) - decoded_data_ben = decode_event_data_message(encoder, event_data_ben) - decoded_data_alice = decode_event_data_message(encoder, event_data_alice) - encoder.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_with_metadata.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_with_metadata.py deleted file mode 100644 index e3975873c064..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_and_decode_with_metadata.py +++ /dev/null @@ -1,91 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import os - -from azure.identity import ClientSecretCredential -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.eventhub import EventData - -TENANT_ID = os.environ["AZURE_TENANT_ID"] -CLIENT_ID = os.environ["AZURE_CLIENT_ID"] -CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ - "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" -] -GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET -) - - -def encode_metadata_dict(encoder): - dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - encoded_metadata_ben = encoder.encode(dict_data_ben, schema=SCHEMA_STRING) - - print("Encoded metadata is: ", encoded_metadata_ben) - return EventData.from_message_data( - encoded_metadata_ben["data"], - encoded_metadata_ben["content_type"], - ) - -def decode_with_data_and_content_type(encoder, event_data): - # get data as bytes - data = bytearray() - for d in event_data.body: - data += d - data_bytes = bytes(data) - data_dict = {"data": data_bytes, "content_type": event_data.content_type} - decoded_data = encoder.decode(data_dict) - - print("Decoded data is: ", decoded_data) - return decoded_data - - -if __name__ == "__main__": - schema_registry = SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=token_credential, - ) - encoder = AvroEncoder( - client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True - ) - event_data = encode_metadata_dict(encoder) - decoded_data = decode_with_data_and_content_type(encoder, event_data) - encoder.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_with_callback.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_with_callback.py deleted file mode 100644 index 8dab085feb1e..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/encode_with_callback.py +++ /dev/null @@ -1,83 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import os - -from azure.identity import ClientSecretCredential -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.eventhub import EventData - -TENANT_ID = os.environ["AZURE_TENANT_ID"] -CLIENT_ID = os.environ["AZURE_CLIENT_ID"] -CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ - "SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE" -] -GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"] -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -token_credential = ClientSecretCredential( - tenant_id=TENANT_ID, client_id=CLIENT_ID, client_secret=CLIENT_SECRET -) - -def encode_with_callback(encoder): - - # Callback MUST have these parameters in the following order: (data, content_type, **kwargs) - def sample_create_event_data(data, content_type, **kwargs): - print("Creating sample EventData with callback.") - return EventData.from_message_data(data, content_type, **kwargs) - - dict_data_ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"} - event_data_ben = encoder.encode( - dict_data_ben, schema=SCHEMA_STRING, message_type=sample_create_event_data - ) - - print("Encoded data is: ", next(event_data_ben.body)) - print("Encoded content type is: ", event_data_ben.content_type) - return event_data_ben - - -if __name__ == "__main__": - schema_registry = SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=token_credential, - ) - encoder = AvroEncoder( - client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True - ) - event_data_ben = encode_with_callback(encoder) - encoder.close() diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_receive_integration.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_receive_integration.py deleted file mode 100644 index 2e070b77846f..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_receive_integration.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -""" -Examples to show receiving events from EventHub with AvroEncoder integrated for data decoding. -""" - -# pylint: disable=C0111 -import os -from azure.eventhub import EventHubConsumerClient -from azure.identity import DefaultAzureCredential -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder - -EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] -EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] - - -def on_event(partition_context, event): - print(f"Received event from partition: {partition_context.partition_id}.") - - bytes_payload = b"".join(b for b in event.body) - print(f'The received bytes of the EventData is {bytes_payload}.') - - # Use the decode method to decode the payload of the event. - # The decode method will extract the schema id from the content_type, and automatically retrieve the Avro Schema - # from the Schema Registry Service. The schema will be cached locally for future usage. - decoded_data = avro_encoder.decode(event) - print(f'The dict data after decoding is {decoded_data}') - - -# create an EventHubConsumerClient instance -eventhub_consumer = EventHubConsumerClient.from_connection_string( - conn_str=EVENTHUB_CONNECTION_STR, - consumer_group='$Default', - eventhub_name=EVENTHUB_NAME, -) - - -# create a AvroEncoder instance -avro_encoder = AvroEncoder( - client=SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=DefaultAzureCredential() - ), - group_name=GROUP_NAME, - auto_register_schemas=True -) - - -try: - with eventhub_consumer, avro_encoder: - eventhub_consumer.receive( - on_event=on_event, - starting_position="-1", # "-1" is from the beginning of the partition. - ) -except KeyboardInterrupt: - print('Stopped receiving.') diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_send_integration.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_send_integration.py deleted file mode 100644 index c3b6c6fde8a5..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/sync_samples/eventhub_send_integration.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -""" -Examples to show sending event to EventHub with AvroEncoder integrated for data encoding. -""" - -# pylint: disable=C0111 - -import os -from azure.eventhub import EventHubProducerClient, EventData -from azure.identity import DefaultAzureCredential -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder - -EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] -EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] - -SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] -GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] - -SCHEMA_STRING = """ -{"namespace": "example.avro", - "type": "record", - "name": "User", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "favorite_number", "type": ["int", "null"]}, - {"name": "favorite_color", "type": ["string", "null"]} - ] -}""" - - -def send_event_data_batch(producer, encoder): - event_data_batch = producer.create_batch() - dict_data = {"name": "Bob", "favorite_number": 7, "favorite_color": "red"} - # Use the encode method to convert dict object to bytes with the given avro schema and set body of EventData. - # The encode method will automatically register the schema into the Schema Registry Service and - # schema will be cached locally for future usage. - event_data = encoder.encode(data=dict_data, schema=SCHEMA_STRING, message_type=EventData) - print(f'The bytes of encoded dict data is {next(event_data.body)}.') - - event_data_batch.add(event_data) - producer.send_batch(event_data_batch) - print('Send is done.') - - -# create an EventHubProducerClient instance -eventhub_producer = EventHubProducerClient.from_connection_string( - conn_str=EVENTHUB_CONNECTION_STR, - eventhub_name=EVENTHUB_NAME -) - - -# create a AvroEncoder instance -avro_encoder = AvroEncoder( - client=SchemaRegistryClient( - fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, - credential=DefaultAzureCredential() - ), - group_name=GROUP_NAME, - auto_register_schemas=True -) - - -with eventhub_producer, avro_encoder: - send_event_data_batch(eventhub_producer, avro_encoder) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/sdk_packaging.toml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/sdk_packaging.toml deleted file mode 100644 index e7687fdae93b..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/sdk_packaging.toml +++ /dev/null @@ -1,2 +0,0 @@ -[packaging] -auto_update = false \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/setup.py deleted file mode 100644 index 5b4b5d7190d3..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/setup.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python - -# ------------------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ------------------------------------------------------------------------- - -import re -import sys -import os.path -from io import open -from setuptools import find_packages, setup - -# Change the PACKAGE_NAME only to change folder and different name -PACKAGE_NAME = "azure-schemaregistry-avroencoder" -PACKAGE_PPRINT_NAME = "Schema Registry Avro Encoder" - -package_folder_path = "azure/schemaregistry/encoder/avroencoder" -namespace_name = "azure.schemaregistry.encoder.avroencoder" - -# Version extraction inspired from 'requests' -with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: - version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', - fd.read(), re.MULTILINE).group(1) - -if not version: - raise RuntimeError('Cannot find version information') - -with open('README.md', encoding='utf-8') as f: - readme = f.read() -with open('CHANGELOG.md', encoding='utf-8') as f: - changelog = f.read() - -exclude_packages = [ - 'tests', - 'samples', - # Exclude packages that will be covered by PEP420 or nspkg - 'azure', - 'azure.schemaregistry', - ] -install_packages = [ - 'azure-schemaregistry>=1.0.0,<2.0.0', - 'avro>=1.11.0' -] - -setup( - name=PACKAGE_NAME, - version=version, - description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME), - long_description=readme + '\n\n' + changelog, - long_description_content_type='text/markdown', - license='MIT License', - author='Microsoft Corporation', - author_email='azpysdkhelp@microsoft.com', - url='https://github.com/Azure/azure-sdk-for-python', - classifiers=[ - "Development Status :: 4 - Beta", - 'Programming Language :: Python', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'License :: OSI Approved :: MIT License', - ], - python_requires=">=3.6", - zip_safe=False, - packages=find_packages(exclude=exclude_packages), - install_requires=install_packages -) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_decode_readers_schema.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_decode_readers_schema.yaml deleted file mode 100644 index 951ab6ce1ee4..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_decode_readers_schema.yaml +++ /dev/null @@ -1,92 +0,0 @@ -interactions: -- request: - body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '201' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:27:57 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Thu, 03 Feb 2022 21:27:57 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_with_auto_register_schemas.yaml deleted file mode 100644 index d82e8ad9c8a4..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_with_auto_register_schemas.yaml +++ /dev/null @@ -1,140 +0,0 @@ -interactions: -- request: - body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '201' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:27:58 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:27:58 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Thu, 03 Feb 2022 21:27:58 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_without_auto_register_schemas.yaml deleted file mode 100644 index 9d7f65944421..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_basic_sr_avro_encoder_without_auto_register_schemas.yaml +++ /dev/null @@ -1,140 +0,0 @@ -interactions: -- request: - body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '201' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:28:00 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:28:00 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Thu, 03 Feb 2022 21:28:01 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_encode_primitive.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_encode_primitive.yaml deleted file mode 100644 index c07893b7cac4..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_encode_primitive.yaml +++ /dev/null @@ -1,48 +0,0 @@ -interactions: -- request: - body: '{"type": "null"}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '16' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/null?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:28:02 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/null/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - bdfae9660e504a13a524d5785901e9a8 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/bdfae9660e504a13a524d5785901e9a8?api-version=2021-10 - schema-name: - - 'null' - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/null/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_encode_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_encode_record.yaml deleted file mode 100644 index bb9c6c707f45..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_encode_record.yaml +++ /dev/null @@ -1,96 +0,0 @@ -interactions: -- request: - body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.populatedrecord\",\n - \ \"type\":\"record\",\n \"fields\":[\n {\"name\":\"name\",\"type\":\"string\"},\n - \ {\"name\":\"age\",\"type\":\"int\"},\n {\"name\":\"married\",\"type\":\"boolean\"},\n - \ {\"name\":\"height\",\"type\":\"float\"},\n {\"name\":\"randb\",\"type\":\"bytes\"}\n - \ ]\n }" - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '405' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.populatedrecord.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:28:04 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 167c05568d1a45bc99cfab05a2a0be37 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/167c05568d1a45bc99cfab05a2a0be37?api-version=2021-10 - schema-name: - - example.avro.populatedrecord.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/167c05568d1a45bc99cfab05a2a0be37?api-version=2021-10 - response: - body: - string: '{"name":"User","namespace":"example.avro.populatedrecord","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"},{"name":"married","type":"boolean"},{"name":"height","type":"float"},{"name":"randb","type":"bytes"}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Thu, 03 Feb 2022 21:28:04 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 167c05568d1a45bc99cfab05a2a0be37 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/167c05568d1a45bc99cfab05a2a0be37?api-version=2021-10 - schema-name: - - example.avro.populatedrecord.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_parse_error_schema_as_record.yaml deleted file mode 100644 index b4ff9bf8b138..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_parse_error_schema_as_record.yaml +++ /dev/null @@ -1,94 +0,0 @@ -interactions: -- request: - body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.error\",\n - \ \"type\":\"error\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '167' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.error.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:28:05 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - b161a83a9f1a41fe8d8c07e94bf7cfeb - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/b161a83a9f1a41fe8d8c07e94bf7cfeb?api-version=2021-10 - schema-name: - - example.avro.error.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/b161a83a9f1a41fe8d8c07e94bf7cfeb?api-version=2021-10 - response: - body: - string: '{"name":"User","namespace":"example.avro.error","type":"error","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Thu, 03 Feb 2022 21:28:06 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - b161a83a9f1a41fe8d8c07e94bf7cfeb - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/b161a83a9f1a41fe8d8c07e94bf7cfeb?api-version=2021-10 - schema-name: - - example.avro.error.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_parse_record_name.yaml deleted file mode 100644 index a4cdfa2917bb..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder.test_parse_record_name.yaml +++ /dev/null @@ -1,185 +0,0 @@ -interactions: -- request: - body: "{\n \"namespace\": \"thrownaway\",\n \"name\":\"User.avro\",\n - \ \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '166' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User.avro?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:28:06 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - ea10fc27c7f34996b6a26a11f08124e4 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/ea10fc27c7f34996b6a26a11f08124e4?api-version=2021-10 - schema-name: - - User.avro - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/ea10fc27c7f34996b6a26a11f08124e4?api-version=2021-10 - response: - body: - string: '{"namespace":"thrownaway","name":"User.avro","type":"record","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Thu, 03 Feb 2022 21:28:08 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - ea10fc27c7f34996b6a26a11f08124e4 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/ea10fc27c7f34996b6a26a11f08124e4?api-version=2021-10 - schema-name: - - User.avro - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -- request: - body: "{\n \"name\":\"User\",\n \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '122' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Thu, 03 Feb 2022 21:28:08 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - de23e62d4e20433bbeac7badad5581a4 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/de23e62d4e20433bbeac7badad5581a4?api-version=2021-10 - schema-name: - - User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/de23e62d4e20433bbeac7badad5581a4?api-version=2021-10 - response: - body: - string: '{"name":"User","type":"record","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Thu, 03 Feb 2022 21:28:09 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - de23e62d4e20433bbeac7badad5581a4 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/de23e62d4e20433bbeac7badad5581a4?api-version=2021-10 - schema-name: - - User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_decode_readers_schema.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_decode_readers_schema.yaml deleted file mode 100644 index 6bab4e9f3c48..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_decode_readers_schema.yaml +++ /dev/null @@ -1,63 +0,0 @@ -interactions: -- request: - body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - Accept: - - application/json - Content-Length: - - '201' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:09 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: application/json;serialization=Avro - date: Thu, 03 Feb 2022 21:28:10 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_with_auto_register_schemas.yaml deleted file mode 100644 index 3b9d1b73fcf7..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_with_auto_register_schemas.yaml +++ /dev/null @@ -1,99 +0,0 @@ -interactions: -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:11 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:11 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User:get-id?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: application/json;serialization=Avro - date: Thu, 03 Feb 2022 21:28:12 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_without_auto_register_schemas.yaml deleted file mode 100644 index da7eea5f67fd..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_basic_sr_avro_encoder_without_auto_register_schemas.yaml +++ /dev/null @@ -1,99 +0,0 @@ -interactions: -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:12 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:12 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User:get-id?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: application/json;serialization=Avro - date: Thu, 03 Feb 2022 21:28:13 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 648a9a85c4c143e8a4723e820300e5ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/648a9a85c4c143e8a4723e820300e5ef?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_encode_primitive.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_encode_primitive.yaml deleted file mode 100644 index 43e7093c9540..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_encode_primitive.yaml +++ /dev/null @@ -1,34 +0,0 @@ -interactions: -- request: - body: '{"type": "null"}' - headers: - Accept: - - application/json - Content-Length: - - '16' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/null?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:14 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/null/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: bdfae9660e504a13a524d5785901e9a8 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/bdfae9660e504a13a524d5785901e9a8?api-version=2021-10 - schema-name: 'null' - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/null/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/null?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_encode_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_encode_record.yaml deleted file mode 100644 index 302b4655ba11..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_encode_record.yaml +++ /dev/null @@ -1,67 +0,0 @@ -interactions: -- request: - body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.populatedrecord\",\n - \ \"type\":\"record\",\n \"fields\":[\n {\"name\":\"name\",\"type\":\"string\"},\n - \ {\"name\":\"age\",\"type\":\"int\"},\n {\"name\":\"married\",\"type\":\"boolean\"},\n - \ {\"name\":\"height\",\"type\":\"float\"},\n {\"name\":\"randb\",\"type\":\"bytes\"}\n - \ ]\n }" - headers: - Accept: - - application/json - Content-Length: - - '405' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.populatedrecord.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:15 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 167c05568d1a45bc99cfab05a2a0be37 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/167c05568d1a45bc99cfab05a2a0be37?api-version=2021-10 - schema-name: example.avro.populatedrecord.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.populatedrecord.User?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/167c05568d1a45bc99cfab05a2a0be37?api-version=2021-10 - response: - body: - string: '{"name":"User","namespace":"example.avro.populatedrecord","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"},{"name":"married","type":"boolean"},{"name":"height","type":"float"},{"name":"randb","type":"bytes"}]}' - headers: - content-type: application/json;serialization=Avro - date: Thu, 03 Feb 2022 21:28:15 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 167c05568d1a45bc99cfab05a2a0be37 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/167c05568d1a45bc99cfab05a2a0be37?api-version=2021-10 - schema-name: example.avro.populatedrecord.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/167c05568d1a45bc99cfab05a2a0be37?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_parse_error_schema_as_record.yaml deleted file mode 100644 index 53b97dd14b45..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_parse_error_schema_as_record.yaml +++ /dev/null @@ -1,65 +0,0 @@ -interactions: -- request: - body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.error\",\n - \ \"type\":\"error\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Content-Length: - - '167' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.error.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:17 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: b161a83a9f1a41fe8d8c07e94bf7cfeb - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/b161a83a9f1a41fe8d8c07e94bf7cfeb?api-version=2021-10 - schema-name: example.avro.error.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.error.User?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/b161a83a9f1a41fe8d8c07e94bf7cfeb?api-version=2021-10 - response: - body: - string: '{"name":"User","namespace":"example.avro.error","type":"error","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: application/json;serialization=Avro - date: Thu, 03 Feb 2022 21:28:17 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: b161a83a9f1a41fe8d8c07e94bf7cfeb - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/b161a83a9f1a41fe8d8c07e94bf7cfeb?api-version=2021-10 - schema-name: example.avro.error.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/b161a83a9f1a41fe8d8c07e94bf7cfeb?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_parse_record_name.yaml deleted file mode 100644 index f8279415b549..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_encoder_async.test_parse_record_name.yaml +++ /dev/null @@ -1,127 +0,0 @@ -interactions: -- request: - body: "{\n \"namespace\": \"thrownaway\",\n \"name\":\"User.avro\",\n - \ \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Content-Length: - - '166' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User.avro?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:18 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: ea10fc27c7f34996b6a26a11f08124e4 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/ea10fc27c7f34996b6a26a11f08124e4?api-version=2021-10 - schema-name: User.avro - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/User.avro?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/ea10fc27c7f34996b6a26a11f08124e4?api-version=2021-10 - response: - body: - string: '{"namespace":"thrownaway","name":"User.avro","type":"record","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: application/json;serialization=Avro - date: Thu, 03 Feb 2022 21:28:19 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: ea10fc27c7f34996b6a26a11f08124e4 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/ea10fc27c7f34996b6a26a11f08124e4?api-version=2021-10 - schema-name: User.avro - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/ea10fc27c7f34996b6a26a11f08124e4?api-version=2021-10 -- request: - body: "{\n \"name\":\"User\",\n \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Content-Length: - - '122' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Thu, 03 Feb 2022 21:28:19 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: de23e62d4e20433bbeac7badad5581a4 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/de23e62d4e20433bbeac7badad5581a4?api-version=2021-10 - schema-name: User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/User?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/de23e62d4e20433bbeac7badad5581a4?api-version=2021-10 - response: - body: - string: '{"name":"User","type":"record","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: application/json;serialization=Avro - date: Thu, 03 Feb 2022 21:28:20 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: de23e62d4e20433bbeac7badad5581a4 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/de23e62d4e20433bbeac7badad5581a4?api-version=2021-10 - schema-name: User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/de23e62d4e20433bbeac7badad5581a4?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml deleted file mode 100644 index 27e450ae7cd6..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml +++ /dev/null @@ -1,140 +0,0 @@ -interactions: -- request: - body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '201' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:01 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 8bed862d53c8482880a546b063f8f6ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:01 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 8bed862d53c8482880a546b063f8f6ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Fri, 05 Nov 2021 23:17:01 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 8bed862d53c8482880a546b063f8f6ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml deleted file mode 100644 index 43c228cb5301..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml +++ /dev/null @@ -1,140 +0,0 @@ -interactions: -- request: - body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '201' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:02 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 8bed862d53c8482880a546b063f8f6ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:03 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 8bed862d53c8482880a546b063f8f6ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Fri, 05 Nov 2021 23:17:03 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 8bed862d53c8482880a546b063f8f6ef - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: - - example.avro.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml deleted file mode 100644 index db54bb0c2f81..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml +++ /dev/null @@ -1,94 +0,0 @@ -interactions: -- request: - body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.error\",\n - \ \"type\":\"error\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '167' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.error.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:05 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 126ed9b72e0941ce9ec06ce47051bc63 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/126ed9b72e0941ce9ec06ce47051bc63?api-version=2021-10 - schema-name: - - example.avro.error.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/126ed9b72e0941ce9ec06ce47051bc63?api-version=2021-10 - response: - body: - string: '{"name":"User","namespace":"example.avro.error","type":"error","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Fri, 05 Nov 2021 23:17:05 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 126ed9b72e0941ce9ec06ce47051bc63 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/126ed9b72e0941ce9ec06ce47051bc63?api-version=2021-10 - schema-name: - - example.avro.error.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_parse_record_name.yaml deleted file mode 100644 index 6fd66e14703e..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_parse_record_name.yaml +++ /dev/null @@ -1,185 +0,0 @@ -interactions: -- request: - body: "{\n \"namespace\": \"thrownaway\",\n \"name\":\"User.avro\",\n - \ \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '166' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User.avro?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:07 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - c785cfbccd2e43a69e0a95e61506da4c - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/c785cfbccd2e43a69e0a95e61506da4c?api-version=2021-10 - schema-name: - - User.avro - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/c785cfbccd2e43a69e0a95e61506da4c?api-version=2021-10 - response: - body: - string: '{"namespace":"thrownaway","name":"User.avro","type":"record","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Fri, 05 Nov 2021 23:17:07 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - c785cfbccd2e43a69e0a95e61506da4c - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/c785cfbccd2e43a69e0a95e61506da4c?api-version=2021-10 - schema-name: - - User.avro - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -- request: - body: "{\n \"name\":\"User\",\n \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '122' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:08 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 31b0db60056a4d43b880870cf7139c72 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/31b0db60056a4d43b880870cf7139c72?api-version=2021-10 - schema-name: - - User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/31b0db60056a4d43b880870cf7139c72?api-version=2021-10 - response: - body: - string: '{"name":"User","type":"record","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Fri, 05 Nov 2021 23:17:08 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - 31b0db60056a4d43b880870cf7139c72 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/31b0db60056a4d43b880870cf7139c72?api-version=2021-10 - schema-name: - - User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml deleted file mode 100644 index 5f149706108c..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml +++ /dev/null @@ -1,48 +0,0 @@ -interactions: -- request: - body: '{"type": "null"}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '16' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/null?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:09 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/null/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - fa716f435d9947039d5eab85735f3fd8 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/fa716f435d9947039d5eab85735f3fd8?api-version=2021-10 - schema-name: - - 'null' - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/null/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_serialize_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_serialize_record.yaml deleted file mode 100644 index d4d59267260b..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer.test_serialize_record.yaml +++ /dev/null @@ -1,96 +0,0 @@ -interactions: -- request: - body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.populatedrecord\",\n - \ \"type\":\"record\",\n \"fields\":[\n {\"name\":\"name\",\"type\":\"string\"},\n - \ {\"name\":\"age\",\"type\":\"int\"},\n {\"name\":\"married\",\"type\":\"boolean\"},\n - \ {\"name\":\"height\",\"type\":\"float\"},\n {\"name\":\"randb\",\"type\":\"bytes\"}\n - \ ]\n }" - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '405' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.populatedrecord.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: - - '0' - date: - - Fri, 05 Nov 2021 23:17:12 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - e151432d25624815913f3fe5e9e90139 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/e151432d25624815913f3fe5e9e90139?api-version=2021-10 - schema-name: - - example.avro.populatedrecord.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - status: - code: 204 - message: No Content -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/e151432d25624815913f3fe5e9e90139?api-version=2021-10 - response: - body: - string: '{"name":"User","namespace":"example.avro.populatedrecord","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"},{"name":"married","type":"boolean"},{"name":"height","type":"float"},{"name":"randb","type":"bytes"}]}' - headers: - content-type: - - application/json;serialization=Avro - date: - - Fri, 05 Nov 2021 23:17:13 GMT - location: - - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 - schema-group-name: - - swathip-test-schema - schema-id: - - e151432d25624815913f3fe5e9e90139 - schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/e151432d25624815913f3fe5e9e90139?api-version=2021-10 - schema-name: - - example.avro.populatedrecord.User - schema-version: - - '1' - schema-versions-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 - server: - - Microsoft-HTTPAPI/2.0 - strict-transport-security: - - max-age=31536000 - transfer-encoding: - - chunked - status: - code: 200 - message: OK -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml deleted file mode 100644 index a4c3927c75a4..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml +++ /dev/null @@ -1,99 +0,0 @@ -interactions: -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:13 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 8bed862d53c8482880a546b063f8f6ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:14 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 8bed862d53c8482880a546b063f8f6ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User:get-id?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: application/json;serialization=Avro - date: Fri, 05 Nov 2021 23:17:14 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 8bed862d53c8482880a546b063f8f6ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml deleted file mode 100644 index 8195aed69c20..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml +++ /dev/null @@ -1,99 +0,0 @@ -interactions: -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:16 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 8bed862d53c8482880a546b063f8f6ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 -- request: - body: '{"type": "record", "name": "User", "namespace": "example.avro", "fields": - [{"type": "string", "name": "name"}, {"type": ["int", "null"], "name": "favorite_number"}, - {"type": ["string", "null"], "name": "favorite_color"}]}' - headers: - Accept: - - application/json - Content-Length: - - '221' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: POST - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User:get-id?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:17 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 8bed862d53c8482880a546b063f8f6ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User:get-id?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - response: - body: - string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' - headers: - content-type: application/json;serialization=Avro - date: Fri, 05 Nov 2021 23:17:17 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 8bed862d53c8482880a546b063f8f6ef - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 - schema-name: example.avro.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/8bed862d53c8482880a546b063f8f6ef?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml deleted file mode 100644 index 28f2f6b27e41..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml +++ /dev/null @@ -1,65 +0,0 @@ -interactions: -- request: - body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.error\",\n - \ \"type\":\"error\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Content-Length: - - '167' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.error.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:18 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 126ed9b72e0941ce9ec06ce47051bc63 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/126ed9b72e0941ce9ec06ce47051bc63?api-version=2021-10 - schema-name: example.avro.error.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.error.User?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/126ed9b72e0941ce9ec06ce47051bc63?api-version=2021-10 - response: - body: - string: '{"name":"User","namespace":"example.avro.error","type":"error","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: application/json;serialization=Avro - date: Fri, 05 Nov 2021 23:17:19 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 126ed9b72e0941ce9ec06ce47051bc63 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/126ed9b72e0941ce9ec06ce47051bc63?api-version=2021-10 - schema-name: example.avro.error.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/126ed9b72e0941ce9ec06ce47051bc63?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml deleted file mode 100644 index 8474a78fb1ad..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml +++ /dev/null @@ -1,127 +0,0 @@ -interactions: -- request: - body: "{\n \"namespace\": \"thrownaway\",\n \"name\":\"User.avro\",\n - \ \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Content-Length: - - '166' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User.avro?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:19 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: c785cfbccd2e43a69e0a95e61506da4c - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/c785cfbccd2e43a69e0a95e61506da4c?api-version=2021-10 - schema-name: User.avro - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/User.avro?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/c785cfbccd2e43a69e0a95e61506da4c?api-version=2021-10 - response: - body: - string: '{"namespace":"thrownaway","name":"User.avro","type":"record","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: application/json;serialization=Avro - date: Fri, 05 Nov 2021 23:17:20 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: c785cfbccd2e43a69e0a95e61506da4c - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/c785cfbccd2e43a69e0a95e61506da4c?api-version=2021-10 - schema-name: User.avro - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/c785cfbccd2e43a69e0a95e61506da4c?api-version=2021-10 -- request: - body: "{\n \"name\":\"User\",\n \"type\":\"record\",\n \"fields\":[{\"name\":\"name\",\"type\":\"string\"}]\n - \ }" - headers: - Accept: - - application/json - Content-Length: - - '122' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:20 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 31b0db60056a4d43b880870cf7139c72 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/31b0db60056a4d43b880870cf7139c72?api-version=2021-10 - schema-name: User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/User?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/31b0db60056a4d43b880870cf7139c72?api-version=2021-10 - response: - body: - string: '{"name":"User","type":"record","fields":[{"name":"name","type":"string"}]}' - headers: - content-type: application/json;serialization=Avro - date: Fri, 05 Nov 2021 23:17:21 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: 31b0db60056a4d43b880870cf7139c72 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/31b0db60056a4d43b880870cf7139c72?api-version=2021-10 - schema-name: User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/31b0db60056a4d43b880870cf7139c72?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml deleted file mode 100644 index 66cd2e2c3875..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml +++ /dev/null @@ -1,34 +0,0 @@ -interactions: -- request: - body: '{"type": "null"}' - headers: - Accept: - - application/json - Content-Length: - - '16' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/null?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:23 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/null/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: fa716f435d9947039d5eab85735f3fd8 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/fa716f435d9947039d5eab85735f3fd8?api-version=2021-10 - schema-name: 'null' - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/null/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/null?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml deleted file mode 100644 index 5474d6fc8054..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml +++ /dev/null @@ -1,67 +0,0 @@ -interactions: -- request: - body: "{\n \"name\":\"User\",\n \"namespace\":\"example.avro.populatedrecord\",\n - \ \"type\":\"record\",\n \"fields\":[\n {\"name\":\"name\",\"type\":\"string\"},\n - \ {\"name\":\"age\",\"type\":\"int\"},\n {\"name\":\"married\",\"type\":\"boolean\"},\n - \ {\"name\":\"height\",\"type\":\"float\"},\n {\"name\":\"randb\",\"type\":\"bytes\"}\n - \ ]\n }" - headers: - Accept: - - application/json - Content-Length: - - '405' - Content-Type: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: PUT - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.populatedrecord.User?api-version=2021-10 - response: - body: - string: '' - headers: - content-length: '0' - date: Fri, 05 Nov 2021 23:17:24 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: e151432d25624815913f3fe5e9e90139 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/e151432d25624815913f3fe5e9e90139?api-version=2021-10 - schema-name: example.avro.populatedrecord.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - status: - code: 204 - message: No Content - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.populatedrecord.User?api-version=2021-10 -- request: - body: null - headers: - Accept: - - application/json; serialization=Avro - User-Agent: - - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) - method: GET - uri: https://fake_resource.servicebus.windows.net/$schemaGroups/$schemas/e151432d25624815913f3fe5e9e90139?api-version=2021-10 - response: - body: - string: '{"name":"User","namespace":"example.avro.populatedrecord","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"},{"name":"married","type":"boolean"},{"name":"height","type":"float"},{"name":"randb","type":"bytes"}]}' - headers: - content-type: application/json;serialization=Avro - date: Fri, 05 Nov 2021 23:17:24 GMT - location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 - schema-group-name: swathip-test-schema - schema-id: e151432d25624815913f3fe5e9e90139 - schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/e151432d25624815913f3fe5e9e90139?api-version=2021-10 - schema-name: example.avro.populatedrecord.User - schema-version: '1' - schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2021-10 - server: Microsoft-HTTPAPI/2.0 - strict-transport-security: max-age=31536000 - transfer-encoding: chunked - status: - code: 200 - message: OK - url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/$schemas/e151432d25624815913f3fe5e9e90139?api-version=2021-10 -version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/test_avro_encoder.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/test_avro_encoder.py deleted file mode 100644 index 8c8d70b2b46f..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/test_avro_encoder.py +++ /dev/null @@ -1,500 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import functools -import pytest -import json - -from azure.schemaregistry import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder import AvroEncoder -from azure.schemaregistry.encoder.avroencoder.exceptions import SchemaParseError, SchemaEncodeError, SchemaDecodeError - -import avro -from avro.errors import AvroTypeException -from azure.schemaregistry.encoder.avroencoder._apache_avro_encoder import ApacheAvroObjectEncoder as AvroObjectEncoder - -from devtools_testutils import AzureTestCase, PowerShellPreparer - -SchemaRegistryPowerShellPreparer = functools.partial(PowerShellPreparer, "schemaregistry", schemaregistry_fully_qualified_namespace="fake_resource.servicebus.windows.net/", schemaregistry_group="fakegroup") - -class AvroEncoderTests(AzureTestCase): - - def test_raw_avro_encoder(self): - schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - schema = avro.schema.parse(schema_str) - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - - raw_avro_object_encoder = AvroObjectEncoder() - - # encoding part - encoded_payload = raw_avro_object_encoder.encode(dict_data, schema_str) - - # decoding part - decoded_data = raw_avro_object_encoder.decode(encoded_payload, schema_str) - - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - assert decoded_data["favorite_color"] == u"red" - - dict_data_missing_optional_fields = {"name": u"Alice"} - encoded_payload = raw_avro_object_encoder.encode(dict_data_missing_optional_fields, schema_str) - decoded_data = raw_avro_object_encoder.decode(encoded_payload, schema_str) - - assert decoded_data["name"] == u"Alice" - assert not decoded_data["favorite_number"] - assert not decoded_data["favorite_color"] - - def test_raw_avro_encoder_negative(self): - schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - schema = avro.schema.parse(schema_str) - - raw_avro_object_encoder = AvroObjectEncoder() - dict_data_wrong_type = {"name": u"Ben", "favorite_number": u"something", "favorite_color": u"red"} - with pytest.raises(AvroTypeException): # avro.io.AvroTypeException - raw_avro_object_encoder.encode(dict_data_wrong_type, schema_str) - - dict_data_missing_required_field = {"favorite_number": 7, "favorite_color": u"red"} - with pytest.raises(AvroTypeException): # avro.io.AvroTypeException - raw_avro_object_encoder.encode(dict_data_missing_required_field, schema_str) - - @SchemaRegistryPowerShellPreparer() - def test_basic_sr_avro_encoder_with_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - schema = avro.schema.parse(schema_str) - - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - encoded_metadata = sr_avro_encoder.encode(dict_data, schema=schema_str) - content_type = encoded_metadata["content_type"] - encoded_data = encoded_metadata["data"] - - assert content_type.split("+")[0] == 'avro/binary' - schema_id = sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro").id - assert content_type.split("+")[1] == schema_id - - encoded_data_dict = {"data": encoded_data, "content_type": content_type} - decoded_data = sr_avro_encoder.decode(encoded_data_dict) - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - assert decoded_data["favorite_color"] == u"red" - - # check that AvroEncoder won't work with message types that don't follow protocols - class BadExample: - def __init__(self, not_data): - self.not_data = not_data - - with pytest.raises(SchemaEncodeError) as e: # caught avro SchemaParseError - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_str, message_type=BadExample) - assert "subtype of the MessageType" in (str(e.value)) - - bad_ex = BadExample('fake') - with pytest.raises(SchemaDecodeError) as e: # caught avro SchemaParseError - sr_avro_encoder.decode(message=bad_ex) - assert "subtype of the MessageType" in (str(e.value)) - - # check that AvroEncoder will work with message types that follow protocols - class GoodExample: - def __init__(self, data: bytes, content_type: str, **kwargs): - self.data = data - self.content_type = content_type - self.extra = kwargs.pop('extra', None) - - def __message_data__(self): - return {"data": self.data, "content_type": self.content_type} - - def good_callback(data: bytes, content_type: str, **kwargs): - return GoodExample(data, content_type, **kwargs) - - good_ex_obj = sr_avro_encoder.encode(dict_data, schema=schema_str, message_type=GoodExample, extra='val') - good_ex_callback = sr_avro_encoder.encode(dict_data, schema=schema_str, message_type=good_callback, extra='val') - decoded_data_obj = sr_avro_encoder.decode(message=good_ex_obj) - decoded_data_callback = sr_avro_encoder.decode(message=good_ex_callback) - - assert decoded_data_obj["name"] == u"Ben" - assert decoded_data_obj["favorite_number"] == 7 - assert decoded_data_obj["favorite_color"] == u"red" - assert decoded_data_callback["name"] == u"Ben" - assert decoded_data_callback["favorite_number"] == 7 - assert decoded_data_callback["favorite_color"] == u"red" - - sr_avro_encoder.close() - - @SchemaRegistryPowerShellPreparer() - def test_basic_sr_avro_encoder_without_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group) - - schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - schema = avro.schema.parse(schema_str) - - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - encoded_metadata = sr_avro_encoder.encode(dict_data, schema=schema_str) - content_type = encoded_metadata["content_type"] - encoded_data = encoded_metadata["data"] - - assert content_type.split("+")[0] == 'avro/binary' - schema_id = sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro").id - assert content_type.split("+")[1] == schema_id - - encoded_data_dict = {"data": encoded_data, "content_type": content_type} - decoded_data = sr_avro_encoder.decode(encoded_data_dict) - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - assert decoded_data["favorite_color"] == u"red" - - sr_avro_encoder.close() - - @SchemaRegistryPowerShellPreparer() - def test_basic_sr_avro_encoder_decode_readers_schema(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - encoded_metadata = sr_avro_encoder.encode(dict_data, schema=schema_str) - content_type = encoded_metadata["content_type"] - encoded_data = encoded_metadata["data"] - - # readers_schema with removed field - readers_schema_remove_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]}]}""" - encoded_data_dict = {"data": encoded_data, "content_type": content_type} - decoded_data = sr_avro_encoder.decode(encoded_data_dict, readers_schema=readers_schema_remove_field) - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - - # readers_schema with extra field with default - readers_schema_extra_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}, {"name":"favorite_city","type":["string","null"], "default": "Redmond"}]}""" - decoded_data = sr_avro_encoder.decode(encoded_data_dict, readers_schema=readers_schema_extra_field) - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - assert decoded_data["favorite_color"] == "red" - assert decoded_data["favorite_city"] == "Redmond" - - # readers_schema with changed name results in error - readers_schema_change_name = """{"namespace":"fakeexample.avro","type":"record","name":"fake_user","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - with pytest.raises(SchemaDecodeError): - decoded_data = sr_avro_encoder.decode(encoded_data_dict, readers_schema=readers_schema_change_name) - - - ################################################################# - ######################### PARSE SCHEMAS ######################### - ################################################################# - - @SchemaRegistryPowerShellPreparer() - def test_parse_invalid_json_string(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - invalid_schema = { - "name":"User", - "type":"record", - "namespace":"example.avro", - "fields":[{"name":"name","type":"string"}] - } - invalid_schema_string = "{}".format(invalid_schema) - with pytest.raises(SchemaParseError): # caught avro SchemaParseError - sr_avro_encoder.encode({"name": u"Ben"}, schema=invalid_schema_string) - - ######################### PRIMITIVES ######################### - - @SchemaRegistryPowerShellPreparer() - def test_parse_primitive_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - primitive_string = "string" - with pytest.raises(SchemaParseError) as e: - sr_avro_encoder.encode("hello", schema=primitive_string) - - ######################### type fixed ######################### - - @SchemaRegistryPowerShellPreparer() - def test_parse_fixed_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - # avro bug: should give warning from IgnoredLogicalType error since precision < 0 - #fixed_type_ignore_logical_type_error = """{"type": "fixed", "size": 4, "namespace":"example.avro", "name":"User", "precision": -1}""" - #sr_avro_encoder.encode({}, schema=fixed_type_ignore_logical_type_error) - - schema_no_size = """{"type": "fixed", "name":"User"}""" - with pytest.raises(SchemaParseError): # caught AvroException - sr_avro_encoder.encode({}, schema=schema_no_size) - - schema_no_name = """{"type": "fixed", "size": 3}""" - with pytest.raises(SchemaParseError): # caught SchemaParseError - sr_avro_encoder.encode({}, schema=schema_no_name) - - schema_wrong_name = """{"type": "fixed", "name": 1, "size": 3}""" - with pytest.raises(SchemaParseError): # caught SchemaParseError - sr_avro_encoder.encode({}, schema=schema_wrong_name) - - schema_wrong_namespace = """{"type": "fixed", "name": "User", "size": 3, "namespace": 1}""" - with pytest.raises(SchemaParseError): # caught SchemaParseError - sr_avro_encoder.encode({}, schema=schema_wrong_namespace) - - ######################### type unspecified ######################### - - @SchemaRegistryPowerShellPreparer() - def test_parse_invalid_type(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_no_type = """{ - "name": "User", - "namespace":"example.avro", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): # caught avro SchemaParseError - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_no_type) - - schema_wrong_type_type = """{ - "name":"User", - "type":1, - "namespace":"example.avro", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_wrong_type_type) - - ######################### RECORD SCHEMA ######################### - - @SchemaRegistryPowerShellPreparer() - def test_parse_record_name(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_name_has_dot = """{ - "namespace": "thrownaway", - "name":"User.avro", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - encoded_schema = sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_name_has_dot) - schema_id = encoded_schema["content_type"].split("+")[1] - registered_schema = sr_client.get_schema(schema_id) - decoded_registered_schema = json.loads(registered_schema.definition) - - # ensure that namespace is saved as part of name before . in registered schema - assert decoded_registered_schema["name"] == "User.avro" - assert decoded_registered_schema["namespace"] == "thrownaway" - - schema_name_no_namespace = """{ - "name":"User", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - encoded_schema = sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_name_no_namespace) - schema_id = encoded_schema["content_type"].split("+")[1] - registered_schema = sr_client.get_schema(schema_id) - decoded_registered_schema = json.loads(registered_schema.definition) - - assert decoded_registered_schema["name"] == "User" - assert "namespace" not in decoded_registered_schema - - schema_invalid_fullname = """{ - "name":"abc", - "type":"record", - "namespace":"9example.avro", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_invalid_fullname) - - schema_invalid_name_in_fullname = """{ - "name":"1abc", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_invalid_name_in_fullname) - - schema_invalid_name_reserved_type = """{ - "name":"record", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_invalid_name_reserved_type) - - schema_wrong_type_name = """{ - "name":1, - "type":"record", - "namespace":"example.avro", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_wrong_type_name) - - schema_no_name = """{ - "namespace":"example.avro", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_no_name) - - @SchemaRegistryPowerShellPreparer() - def test_parse_error_schema_as_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_error_type = """{ - "name":"User", - "namespace":"example.avro.error", - "type":"error", - "fields":[{"name":"name","type":"string"}] - }""" - encoded_metadata = sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_error_type) - schema_id = encoded_metadata["content_type"].split("+")[1] - registered_schema = sr_client.get_schema(schema_id) - decoded_registered_schema = json.loads(registered_schema.definition) - assert decoded_registered_schema["type"] == "error" - - @SchemaRegistryPowerShellPreparer() - def test_parse_record_fields(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_no_fields = """{ - "name":"User", - "namespace":"example.avro", - "type":"record" - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_no_fields) - - schema_wrong_type_fields = """{ - "name":"User", - "namespace":"example.avro", - "type":"record" - "fields": "hello" - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_wrong_type_fields) - - schema_wrong_field_item_type = """{ - "name":"User", - "namespace":"example.avro", - "type":"record" - "fields": ["hello"] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_wrong_field_item_type) - - schema_record_field_no_name= """{ - "name":"User", - "namespace":"example.avro", - "type":"record", - "fields":[{"type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_record_field_no_name) - - schema_record_field_wrong_type_name= """{ - "name":"User", - "namespace":"example.avro", - "type":"record", - "fields":[{"name": 1, "type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_record_field_wrong_type_name) - - schema_record_field_with_invalid_order = """{ - "name":"User", - "namespace":"example.avro.order", - "type":"record", - "fields":[{"name":"name","type":"string","order":"fake_order"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_record_field_with_invalid_order) - - schema_record_duplicate_fields = """{ - "name":"User", - "namespace":"example.avro", - "type":"record", - "fields":[{"name":"name","type":"string"}, {"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_record_duplicate_fields) - - schema_field_type_invalid = """{ - "name":"User", - "namespace":"example.avro", - "type":"record", - "fields":[{"name":"name","type":1}] - }""" - with pytest.raises(SchemaParseError): - sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_field_type_invalid) - - ################################################################# - #################### SERIALIZE AND DESERIALIZE ################## - ################################################################# - - @SchemaRegistryPowerShellPreparer() - def test_encode_primitive(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - null_type = """{"type": "null"}""" - encoded_metadata = sr_avro_encoder.encode(None, schema=null_type) - assert len(encoded_metadata["data"]) == 0 # assert no data encoded - - @SchemaRegistryPowerShellPreparer() - def test_encode_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - # add below to schema fields later if needed - # {"name":"example.innerrec","type":"record","fields":[{"name":"a","type":"int"}]}, - # {"name":"innerenum","type":"enum","symbols":["FOO", "BAR"]}, - # {"name":"innerarray","type":"array","items":"int"}, - # {"name":"innermap","type":"map","values":"int"}, - # {"name":"innerfixed","type":"fixed","size":74} - schema_record = """{ - "name":"User", - "namespace":"example.avro.populatedrecord", - "type":"record", - "fields":[ - {"name":"name","type":"string"}, - {"name":"age","type":"int"}, - {"name":"married","type":"boolean"}, - {"name":"height","type":"float"}, - {"name":"randb","type":"bytes"} - ] - }""" - data = { - "name": u"Ben", - "age": 3, - "married": False, - "height": 13.5, - "randb": b"\u00FF" - } - - encoded_metadata = sr_avro_encoder.encode(data, schema=schema_record) - decoded_data = sr_avro_encoder.decode(encoded_metadata) - assert decoded_data == data diff --git a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/test_avro_encoder_async.py b/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/test_avro_encoder_async.py deleted file mode 100644 index d8896e1969d6..000000000000 --- a/sdk/schemaregistry/azure-schemaregistry-avroencoder/tests/test_avro_encoder_async.py +++ /dev/null @@ -1,476 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import functools -import pytest -import uuid -import json -from io import BytesIO -import pytest - -import avro -from avro.errors import AvroTypeException - - -from azure.schemaregistry.aio import SchemaRegistryClient -from azure.schemaregistry.encoder.avroencoder.aio import AvroEncoder -from azure.schemaregistry.encoder.avroencoder.exceptions import SchemaParseError, SchemaEncodeError, SchemaDecodeError - -from devtools_testutils import AzureTestCase, PowerShellPreparer - -SchemaRegistryPowerShellPreparer = functools.partial(PowerShellPreparer, "schemaregistry", schemaregistry_fully_qualified_namespace="fake_resource.servicebus.windows.net/", schemaregistry_group="fakegroup") - -class AvroEncoderAsyncTests(AzureTestCase): - - def create_client(self, fully_qualified_namespace): - credential = self.get_credential(SchemaRegistryClient, is_async=True) - return self.create_client_from_credential(SchemaRegistryClient, credential, fully_qualified_namespace=fully_qualified_namespace, is_async=True) - - @pytest.mark.asyncio - @SchemaRegistryPowerShellPreparer() - async def test_basic_sr_avro_encoder_with_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - async with sr_client: - schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - schema_str = "{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], \"name\": \"favorite_color\"}]}" - schema = avro.schema.parse(schema_str) - - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - encoded_metadata = await sr_avro_encoder.encode(dict_data, schema=schema_str) - content_type = encoded_metadata["content_type"] - encoded_data = encoded_metadata["data"] - - assert content_type.split("+")[0] == 'avro/binary' - schema_properties = await sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro") - schema_id = schema_properties.id - assert content_type.split("+")[1] == schema_id - - encoded_data_dict = {"data": encoded_data, "content_type": content_type} - decoded_data = await sr_avro_encoder.decode(encoded_data_dict) - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - assert decoded_data["favorite_color"] == u"red" - - # check that AvroEncoder won't work with message types that don't follow protocols - class BadExample: - def __init__(self, not_data): - self.not_data = not_data - - with pytest.raises(SchemaEncodeError) as e: # caught avro SchemaParseError - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_str, message_type=BadExample) - assert "subtype of the MessageType" in (str(e.value)) - - bad_ex = BadExample('fake') - with pytest.raises(SchemaDecodeError) as e: # caught avro SchemaParseError - await sr_avro_encoder.decode(message=bad_ex) - assert "subtype of the MessageType" in (str(e.value)) - - # check that AvroEncoder will work with message types that follow protocols - class GoodExample: - def __init__(self, data: bytes, content_type: str, **kwargs): - self.data = data - self.content_type = content_type - self.extra = kwargs.pop('extra', None) - - def __message_data__(self): - return {"data": self.data, "content_type": self.content_type} - - def good_callback(data: bytes, content_type: str, **kwargs): - return GoodExample(data, content_type, **kwargs) - - good_ex_obj = await sr_avro_encoder.encode(dict_data, schema=schema_str, message_type=GoodExample, extra='val') - good_ex_callback = await sr_avro_encoder.encode(dict_data, schema=schema_str, message_type=good_callback, extra='val') - decoded_data_obj = await sr_avro_encoder.decode(message=good_ex_obj) - decoded_data_callback = await sr_avro_encoder.decode(message=good_ex_callback) - - assert decoded_data_obj["name"] == u"Ben" - assert decoded_data_obj["favorite_number"] == 7 - assert decoded_data_obj["favorite_color"] == u"red" - assert decoded_data_callback["name"] == u"Ben" - assert decoded_data_callback["favorite_number"] == 7 - assert decoded_data_callback["favorite_color"] == u"red" - - @pytest.mark.asyncio - @SchemaRegistryPowerShellPreparer() - async def test_basic_sr_avro_encoder_without_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - async with sr_client: - schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - schema_str = "{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], \"name\": \"favorite_color\"}]}" - schema = avro.schema.parse(schema_str) - - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - encoded_metadata = await sr_avro_encoder.encode(dict_data, schema=schema_str) - content_type = encoded_metadata["content_type"] - encoded_data = encoded_metadata["data"] - - assert content_type.split("+")[0] == 'avro/binary' - schema_properties = await sr_client.get_schema_properties(schemaregistry_group, schema.fullname, str(schema), "Avro") - schema_id = schema_properties.id - assert content_type.split("+")[1] == schema_id - - encoded_data_dict = {"data": encoded_data, "content_type": content_type} - decoded_data = await sr_avro_encoder.decode(encoded_data_dict) - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - assert decoded_data["favorite_color"] == u"red" - - @pytest.mark.asyncio - @SchemaRegistryPowerShellPreparer() - async def test_basic_sr_avro_encoder_decode_readers_schema(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - encoded_metadata = await sr_avro_encoder.encode(dict_data, schema=schema_str) - content_type = encoded_metadata["content_type"] - encoded_data = encoded_metadata["data"] - - # readers_schema with removed field - readers_schema_remove_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]}]}""" - encoded_data_dict = {"data": encoded_data, "content_type": content_type} - decoded_data = await sr_avro_encoder.decode(encoded_data_dict, readers_schema=readers_schema_remove_field) - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - - # readers_schema with extra field with default - readers_schema_extra_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}, {"name":"favorite_city","type":["string","null"], "default": "Redmond"}]}""" - encoded_data_dict = {"data": encoded_data, "content_type": content_type} - decoded_data = await sr_avro_encoder.decode(encoded_data_dict, readers_schema=readers_schema_extra_field) - assert decoded_data["name"] == u"Ben" - assert decoded_data["favorite_number"] == 7 - assert decoded_data["favorite_color"] == "red" - assert decoded_data["favorite_city"] == "Redmond" - - # readers_schema with changed name results in error - readers_schema_change_name = """{"namespace":"fakeexample.avro","type":"record","name":"fake_user","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - with pytest.raises(SchemaDecodeError): - encoded_data_dict = {"data": encoded_data, "content_type": content_type} - decoded_data = await sr_avro_encoder.decode(encoded_data_dict, readers_schema=readers_schema_change_name) - print(decoded_data) - - - ################################################################# - ######################### PARSE SCHEMAS ######################### - ################################################################# - - @SchemaRegistryPowerShellPreparer() - async def test_parse_invalid_json_string(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - invalid_schema = { - "name":"User", - "type":"record", - "namespace":"example.avro", - "fields":[{"name":"name","type":"string"}] - } - invalid_schema_string = "{}".format(invalid_schema) - with pytest.raises(SchemaParseError): # caught avro SchemaParseError - await sr_avro_encoder.encode({"name": u"Ben"}, schema=invalid_schema_string) - - ######################### PRIMITIVES ######################### - - @SchemaRegistryPowerShellPreparer() - async def test_parse_primitive_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - primitive_string = "string" - with pytest.raises(SchemaParseError) as e: - await sr_avro_encoder.encode("hello", schema=primitive_string) - - ######################### type fixed ######################### - - @SchemaRegistryPowerShellPreparer() - async def test_parse_fixed_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - # avro bug: should give warning from IgnoredLogicalType error since precision < 0 - #fixed_type_ignore_logical_type_error = """{"type": "fixed", "size": 4, "namespace":"example.avro", "name":"User", "precision": -1}""" - #await sr_avro_encoder.encode({}, schema=fixed_type_ignore_logical_type_error) - - schema_no_size = """{"type": "fixed", "name":"User"}""" - with pytest.raises(SchemaParseError): # caught AvroException - await sr_avro_encoder.encode({}, schema=schema_no_size) - - schema_no_name = """{"type": "fixed", "size": 3}""" - with pytest.raises(SchemaParseError): # caught SchemaParseError - await sr_avro_encoder.encode({}, schema=schema_no_name) - - schema_wrong_name = """{"type": "fixed", "name": 1, "size": 3}""" - with pytest.raises(SchemaParseError): # caught SchemaParseError - await sr_avro_encoder.encode({}, schema=schema_wrong_name) - - schema_wrong_namespace = """{"type": "fixed", "name": "User", "size": 3, "namespace": 1}""" - with pytest.raises(SchemaParseError): # caught SchemaParseError - await sr_avro_encoder.encode({}, schema=schema_wrong_namespace) - - ######################### type unspecified ######################### - - @SchemaRegistryPowerShellPreparer() - async def test_parse_invalid_type(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_no_type = """{ - "name": "User", - "namespace":"example.avro", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): # caught avro SchemaParseError - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_no_type) - - schema_wrong_type_type = """{ - "name":"User", - "type":1, - "namespace":"example.avro", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_wrong_type_type) - - ######################### RECORD SCHEMA ######################### - - @SchemaRegistryPowerShellPreparer() - async def test_parse_record_name(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_name_has_dot = """{ - "namespace": "thrownaway", - "name":"User.avro", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - encoded_schema = await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_name_has_dot) - schema_id = encoded_schema["content_type"].split("+")[1] - registered_schema = await sr_client.get_schema(schema_id) - decoded_registered_schema = json.loads(registered_schema.definition) - - assert decoded_registered_schema["name"] == "User.avro" - assert decoded_registered_schema["namespace"] == "thrownaway" - - schema_name_no_namespace = """{ - "name":"User", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - encoded_schema = await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_name_no_namespace) - schema_id = encoded_schema["content_type"].split("+")[1] - registered_schema = await sr_client.get_schema(schema_id) - decoded_registered_schema = json.loads(registered_schema.definition) - - assert decoded_registered_schema["name"] == "User" - assert "namespace" not in decoded_registered_schema - - schema_invalid_fullname = """{ - "name":"abc", - "type":"record", - "namespace":"9example.avro", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_invalid_fullname) - - schema_invalid_name_in_fullname = """{ - "name":"1abc", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_invalid_name_in_fullname) - - schema_invalid_name_reserved_type = """{ - "name":"record", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_invalid_name_reserved_type) - - schema_wrong_type_name = """{ - "name":1, - "type":"record", - "namespace":"example.avro", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_wrong_type_name) - - schema_no_name = """{ - "namespace":"example.avro", - "type":"record", - "fields":[{"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_no_name) - - @SchemaRegistryPowerShellPreparer() - async def test_parse_error_schema_as_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_error_type = """{ - "name":"User", - "namespace":"example.avro.error", - "type":"error", - "fields":[{"name":"name","type":"string"}] - }""" - encoded_metadata = await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_error_type) - schema_id = encoded_metadata["content_type"].split("+")[1] - registered_schema = await sr_client.get_schema(schema_id) - decoded_registered_schema = json.loads(registered_schema.definition) - assert decoded_registered_schema["type"] == "error" - - @SchemaRegistryPowerShellPreparer() - async def test_parse_record_fields(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - schema_no_fields = """{ - "name":"User", - "namespace":"example.avro", - "type":"record" - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_no_fields) - - schema_wrong_type_fields = """{ - "name":"User", - "namespace":"example.avro", - "type":"record" - "fields": "hello" - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_wrong_type_fields) - - schema_wrong_field_item_type = """{ - "name":"User", - "namespace":"example.avro", - "type":"record" - "fields": ["hello"] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_wrong_field_item_type) - - schema_record_field_no_name= """{ - "name":"User", - "namespace":"example.avro", - "type":"record", - "fields":[{"type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_record_field_no_name) - - schema_record_field_wrong_type_name= """{ - "name":"User", - "namespace":"example.avro", - "type":"record", - "fields":[{"name": 1, "type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_record_field_wrong_type_name) - - schema_record_field_with_invalid_order = """{ - "name":"User", - "namespace":"example.avro.order", - "type":"record", - "fields":[{"name":"name","type":"string","order":"fake_order"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_record_field_with_invalid_order) - - schema_record_duplicate_fields = """{ - "name":"User", - "namespace":"example.avro", - "type":"record", - "fields":[{"name":"name","type":"string"}, {"name":"name","type":"string"}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_record_duplicate_fields) - - schema_field_type_invalid = """{ - "name":"User", - "namespace":"example.avro", - "type":"record", - "fields":[{"name":"name","type":1}] - }""" - with pytest.raises(SchemaParseError): - await sr_avro_encoder.encode({"name": u"Ben"}, schema=schema_field_type_invalid) - - ################################################################# - #################### ENCODE AND DECODE ########################## - ################################################################# - - @SchemaRegistryPowerShellPreparer() - async def test_encode_primitive(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - null_type = """{"type": "null"}""" - encoded_metadata = await sr_avro_encoder.encode(None, schema=null_type) - assert len(encoded_metadata["data"]) == 0 # assert no data encoded - - @SchemaRegistryPowerShellPreparer() - async def test_encode_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): - sr_client = self.create_client(fully_qualified_namespace=schemaregistry_fully_qualified_namespace) - sr_avro_encoder = AvroEncoder(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) - - # add below to schema later if possible - # {"name":"example.innerrec","type":"record","fields":[{"name":"a","type":"int"}]}, - # {"name":"innerenum","type":"enum","symbols":["FOO", "BAR"]}, - # {"name":"innerarray","type":"array","items":"int"}, - # {"name":"innermap","type":"map","values":"int"}, - # {"name":"innerfixed","type":"fixed","size":74} - schema_record = """{ - "name":"User", - "namespace":"example.avro.populatedrecord", - "type":"record", - "fields":[ - {"name":"name","type":"string"}, - {"name":"age","type":"int"}, - {"name":"married","type":"boolean"}, - {"name":"height","type":"float"}, - {"name":"randb","type":"bytes"} - ] - }""" - data = { - "name": u"Ben", - "age": 3, - "married": False, - "height": 13.5, - "randb": b"\u00FF" - } - - encoded_metadata = await sr_avro_encoder.encode(data, schema=schema_record) - decoded_data = await sr_avro_encoder.decode(encoded_metadata) - assert decoded_data == data From a87491e2e6176cda3929aaec3a2efe296f7ba84d Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 25 Feb 2022 08:46:31 -0800 Subject: [PATCH 3/5] add client request kwargs --- .../_schema_registry_avro_serializer.py | 12 ++++++++---- .../aio/_schema_registry_avro_serializer_async.py | 14 ++++++++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index afdb959042e0..173c5c8b7af8 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -130,6 +130,7 @@ def serialize( *, schema: str, message_type: Optional[Callable] = None, + client_request_kwargs: Dict[str, Any] = None, **kwargs: Any, ) -> Union[MessageType, MessageMetadataDict]: """ @@ -154,6 +155,8 @@ def serialize( `(data: bytes, content_type: str, **kwargs) -> MessageType`, where `data` and `content_type` are positional parameters. :paramtype message_type: Callable or None + :keyword client_request_kwargs: The keyword arguments to be passed to the SchemaRegistryClient. + :paramtype client_request_kwargs: Dict[str, Any] :rtype: MessageType or MessageMetadataDict :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaParseError: Indicates an issue with parsing schema. @@ -170,7 +173,7 @@ def serialize( f"Cannot parse schema: {raw_input_schema}", error=e ).raise_with_traceback() - schema_id = self._get_schema_id(schema_fullname, raw_input_schema) + schema_id = self._get_schema_id(schema_fullname, raw_input_schema, **client_request_kwargs) content_type = f"{AVRO_MIME_TYPE}+{schema_id}" try: @@ -225,7 +228,7 @@ def deserialize( message: Union[MessageType, MessageMetadataDict], *, readers_schema: Optional[str] = None, - **kwargs, # pylint: disable=unused-argument + client_request_kwargs: Dict[str, Any] = None, ) -> Dict[str, Any]: """ Deserialize bytes data using schema ID in the content type field. `message` must be one of the following: @@ -267,14 +270,15 @@ def deserialize( data, content_type = self._convert_preamble_format(data, content_type) schema_id = content_type.split("+")[1] - schema_definition = self._get_schema(schema_id) + schema_definition = self._get_schema(schema_id, **client_request_kwargs) try: dict_value = self._avro_serializer.deserialize( data, schema_definition, readers_schema=readers_schema ) except Exception as e: # pylint:disable=broad-except error_message = ( - f"Cannot deserialize value '{data}' for schema: {schema_definition}\n and reader's schema: {readers_schema}" + f"Cannot deserialize value '{data}' for schema: {schema_definition}\n and" + "reader's schema: {readers_schema}" if readers_schema else f"Cannot deserialize value '{data}' for schema: {schema_definition}" ) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py index ce472a54a717..dc726133f8a6 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py @@ -126,6 +126,7 @@ async def serialize( *, schema: str, message_type: Optional[Callable] = None, + client_request_kwargs: Dict[str, Any] = None, **kwargs: Any, ) -> Union[MessageType, MessageMetadataDict]: @@ -151,6 +152,8 @@ async def serialize( `(data: bytes, content_type: str, **kwargs) -> MessageType`, where `data` and `content_type` are positional parameters. :paramtype message_type: Callable or None + :keyword client_request_kwargs: The keyword arguments to be passed to the SchemaRegistryClient. + :paramtype client_request_kwargs: Dict[str, Any] :rtype: MessageType or MessageMetadataDict :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaParseError: Indicates an issue with parsing schema. @@ -167,7 +170,7 @@ async def serialize( f"Cannot parse schema: {raw_input_schema}", error=e ).raise_with_traceback() - schema_id = await self._get_schema_id(schema_fullname, raw_input_schema) + schema_id = await self._get_schema_id(schema_fullname, raw_input_schema, **client_request_kwargs) content_type = f"{AVRO_MIME_TYPE}+{schema_id}" try: @@ -219,7 +222,7 @@ async def deserialize( message: Union[MessageType, MessageMetadataDict], *, readers_schema: Optional[str] = None, - **kwargs, # pylint: disable=unused-argument + client_request_kwargs: Dict[str, Any] = None, ) -> Dict[str, Any]: """ Deserialize bytes data using schema ID in the content type field. `message` must be one of the following: @@ -235,6 +238,8 @@ async def deserialize( :type message: MessageType or MessageMetadataDict :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. :paramtype readers_schema: str or None + :keyword client_request_kwargs: The keyword arguments to be passed to the SchemaRegistryClient. + :paramtype client_request_kwargs: Dict[str, Any] :rtype: Dict[str, Any] :raises ~azure.schemaregistry.serializer.avroserializer.exceptions.SchemaParseError: Indicates an issue with parsing schema. @@ -261,12 +266,13 @@ async def deserialize( data, content_type = self._convert_preamble_format(data, content_type) schema_id = content_type.split("+")[1] - schema_definition = await self._get_schema(schema_id) + schema_definition = await self._get_schema(schema_id, **client_request_kwargs) try: dict_value = self._avro_serializer.deserialize(data, schema_definition, readers_schema=readers_schema) except Exception as e: # pylint:disable=broad-except error_message = ( - f"Cannot deserialize value '{data}' for schema: {schema_definition}\n and reader's schema: {readers_schema}" + f"Cannot deserialize value '{data}' for schema: {schema_definition}\n and" + "reader's schema: {readers_schema}" if readers_schema else f"Cannot deserialize value '{data}' for schema: {schema_definition}" ) From 1407f1d5fc120a7f8676e6fb3b3b77908f149472 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 25 Feb 2022 15:05:52 -0800 Subject: [PATCH 4/5] log if entry added to cache --- .../CHANGELOG.md | 2 ++ .../_schema_registry_avro_serializer.py | 19 +++++++++++++++++++ .../_schema_registry_avro_serializer_async.py | 17 +++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md index 9db5c4105efc..231169031fa7 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md @@ -6,6 +6,8 @@ - APIs have been updated to allow for serializing directly to and deserializing from message type objects, where the data value is the Avro serialized payload. - The content type of the message will hold the schema ID and record format indicator. +- `client_request_kwargs` has been added to `serialize` and `deserialize` on `AvroSerializer` as an optional parameter to be passed into client requests. +- The size of the current schema/schema ID caches will be logged at an info level when a new entry has been added. ### Other Changes diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index 173c5c8b7af8..df376c6c529b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -23,6 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +import logging from functools import lru_cache from io import BytesIO from typing import Any, Dict, Mapping, Optional, Union, Callable @@ -42,6 +43,7 @@ RECORD_FORMAT_IDENTIFIER_LENGTH, ) +_LOGGER = logging.getLogger(__name__) class AvroSerializer(object): """ @@ -173,7 +175,15 @@ def serialize( f"Cannot parse schema: {raw_input_schema}", error=e ).raise_with_traceback() + cache_misses = self._get_schema_id.cache_info().misses # pylint: disable=no-value-for-parameter + client_request_kwargs = client_request_kwargs or {} schema_id = self._get_schema_id(schema_fullname, raw_input_schema, **client_request_kwargs) + new_cache_misses = self._get_schema_id.cache_info().misses # pylint: disable=no-value-for-parameter + if new_cache_misses > cache_misses: + # if more misses, "missed collision" and new element has been added to cache + cache_size = self._get_schema_id.cache_info().currsize # pylint: disable=no-value-for-parameter + _LOGGER.info("New entry has been added to schema ID cache. Cache size: %s", str(cache_size)) + content_type = f"{AVRO_MIME_TYPE}+{schema_id}" try: @@ -270,7 +280,16 @@ def deserialize( data, content_type = self._convert_preamble_format(data, content_type) schema_id = content_type.split("+")[1] + + cache_misses = self._get_schema.cache_info().misses # pylint: disable=no-value-for-parameter + client_request_kwargs = client_request_kwargs or {} schema_definition = self._get_schema(schema_id, **client_request_kwargs) + new_cache_misses = self._get_schema.cache_info().misses # pylint: disable=no-value-for-parameter + if new_cache_misses > cache_misses: + # if more misses, "missed collision" and new element has been added to cache + cache_size = self._get_schema.cache_info().currsize # pylint: disable=no-value-for-parameter + _LOGGER.info("New entry has been added to schema cache. Cache size: %s", str(cache_size)) + try: dict_value = self._avro_serializer.deserialize( data, schema_definition, readers_schema=readers_schema diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py index dc726133f8a6..b5c1aef1e15e 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/aio/_schema_registry_avro_serializer_async.py @@ -23,6 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +import logging from io import BytesIO from typing import Any, Callable, Dict, Mapping, Union, Optional from ._async_lru import alru_cache @@ -41,6 +42,7 @@ SchemaDeserializationError, ) +_LOGGER = logging.getLogger(__name__) class AvroSerializer(object): """ @@ -170,7 +172,14 @@ async def serialize( f"Cannot parse schema: {raw_input_schema}", error=e ).raise_with_traceback() + cache_misses = self._get_schema_id.cache_info().misses # pylint: disable=no-value-for-parameter disable=no-member + client_request_kwargs = client_request_kwargs or {} schema_id = await self._get_schema_id(schema_fullname, raw_input_schema, **client_request_kwargs) + new_cache_misses = self._get_schema_id.cache_info().misses # pylint: disable=no-value-for-parameter disable=no-member + if new_cache_misses > cache_misses: + # if more misses, "missed collision" and new element has been added to cache + cache_size = self._get_schema_id.cache_info().currsize # pylint: disable=no-value-for-parameter disable=no-member + _LOGGER.info("New entry has been added to schema ID cache. Cache size: %s", str(cache_size)) content_type = f"{AVRO_MIME_TYPE}+{schema_id}" try: @@ -266,7 +275,15 @@ async def deserialize( data, content_type = self._convert_preamble_format(data, content_type) schema_id = content_type.split("+")[1] + cache_misses = self._get_schema.cache_info().misses # pylint: disable=no-value-for-parameter disable=no-member + client_request_kwargs = client_request_kwargs or {} schema_definition = await self._get_schema(schema_id, **client_request_kwargs) + new_cache_misses = self._get_schema.cache_info().misses # pylint: disable=no-value-for-parameter disable=no-member + if new_cache_misses > cache_misses: + # if more misses, "missed collision" and new element has been added to cache + cache_size = self._get_schema.cache_info().currsize # pylint: disable=no-value-for-parameter disable=no-member + _LOGGER.info("New entry has been added to schema cache. Cache size: %s", str(cache_size)) + try: dict_value = self._avro_serializer.deserialize(data, schema_definition, readers_schema=readers_schema) except Exception as e: # pylint:disable=broad-except From 48387eaf2f8aa8841f7f4c1846a4a87b33efda97 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 25 Feb 2022 15:58:41 -0800 Subject: [PATCH 5/5] add test for client request kwargs --- ...serializer_with_client_request_kwargs.yaml | 48 ++++++++++++ ...serializer_deserialize_readers_schema.yaml | 4 +- ...serializer_with_auto_register_schemas.yaml | 6 +- ...ializer_without_auto_register_schemas.yaml | 6 +- ...zer.test_parse_error_schema_as_record.yaml | 4 +- ...vro_serializer.test_parse_record_name.yaml | 8 +- ...o_serializer.test_serialize_primitive.yaml | 2 +- ...avro_serializer.test_serialize_record.yaml | 4 +- ...serializer_with_client_request_kwargs.yaml | 34 +++++++++ ...serializer_deserialize_readers_schema.yaml | 4 +- ...serializer_with_auto_register_schemas.yaml | 6 +- ...ializer_without_auto_register_schemas.yaml | 6 +- ...ync.test_parse_error_schema_as_record.yaml | 4 +- ...rializer_async.test_parse_record_name.yaml | 8 +- ...alizer_async.test_serialize_primitive.yaml | 2 +- ...erializer_async.test_serialize_record.yaml | 4 +- .../tests/test_avro_serializer.py | 21 ++++++ .../tests/test_avro_serializer_async.py | 75 ++++++++++++------- 18 files changed, 186 insertions(+), 60 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_avro_serializer_with_client_request_kwargs.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_avro_serializer_with_client_request_kwargs.yaml diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_avro_serializer_with_client_request_kwargs.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_avro_serializer_with_client_request_kwargs.yaml new file mode 100644 index 000000000000..d2208d4f069e --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_avro_serializer_with_client_request_kwargs.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '201' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 25 Feb 2022 23:54:05 GMT + location: + - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: + - swathip-test-schema + schema-id: + - 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: + - example.avro.User + schema-version: + - '1' + schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + status: + code: 204 + message: No Content +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml index 3911d936188e..faba78463382 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml @@ -23,7 +23,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:45 GMT + - Fri, 25 Feb 2022 23:54:06 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: @@ -65,7 +65,7 @@ interactions: content-type: - application/json;serialization=Avro date: - - Fri, 25 Feb 2022 00:47:46 GMT + - Fri, 25 Feb 2022 23:54:07 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml index 65e3279983f3..977ffdcac826 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml @@ -23,7 +23,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:47 GMT + - Fri, 25 Feb 2022 23:54:08 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: @@ -71,7 +71,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:48 GMT + - Fri, 25 Feb 2022 23:54:09 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: @@ -113,7 +113,7 @@ interactions: content-type: - application/json;serialization=Avro date: - - Fri, 25 Feb 2022 00:47:48 GMT + - Fri, 25 Feb 2022 23:54:09 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml index da00b82c35b5..5d5dbec37f55 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml @@ -23,7 +23,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:50 GMT + - Fri, 25 Feb 2022 23:54:10 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: @@ -71,7 +71,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:51 GMT + - Fri, 25 Feb 2022 23:54:11 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: @@ -113,7 +113,7 @@ interactions: content-type: - application/json;serialization=Avro date: - - Fri, 25 Feb 2022 00:47:51 GMT + - Fri, 25 Feb 2022 23:54:11 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml index b21312924136..b1d7baca5980 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml @@ -25,7 +25,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:53 GMT + - Fri, 25 Feb 2022 23:54:13 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 schema-group-name: @@ -67,7 +67,7 @@ interactions: content-type: - application/json;serialization=Avro date: - - Fri, 25 Feb 2022 00:47:54 GMT + - Fri, 25 Feb 2022 23:54:13 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 schema-group-name: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml index b9c2981c5b3b..881c717a5db6 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml @@ -25,7 +25,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:56 GMT + - Fri, 25 Feb 2022 23:54:15 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 schema-group-name: @@ -67,7 +67,7 @@ interactions: content-type: - application/json;serialization=Avro date: - - Fri, 25 Feb 2022 00:47:56 GMT + - Fri, 25 Feb 2022 23:54:15 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 schema-group-name: @@ -116,7 +116,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:57 GMT + - Fri, 25 Feb 2022 23:54:16 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 schema-group-name: @@ -158,7 +158,7 @@ interactions: content-type: - application/json;serialization=Avro date: - - Fri, 25 Feb 2022 00:47:57 GMT + - Fri, 25 Feb 2022 23:54:16 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 schema-group-name: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml index 48ed1fec9c36..ccd98589324b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_primitive.yaml @@ -23,7 +23,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:47:59 GMT + - Fri, 25 Feb 2022 23:54:18 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/null/versions/1?api-version=2021-10 schema-group-name: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml index d6cf017653f9..45cc7406a386 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml @@ -27,7 +27,7 @@ interactions: content-length: - '0' date: - - Fri, 25 Feb 2022 00:48:02 GMT + - Fri, 25 Feb 2022 23:54:19 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 schema-group-name: @@ -69,7 +69,7 @@ interactions: content-type: - application/json;serialization=Avro date: - - Fri, 25 Feb 2022 00:48:02 GMT + - Fri, 25 Feb 2022 23:54:20 GMT location: - https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 schema-group-name: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_avro_serializer_with_client_request_kwargs.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_avro_serializer_with_client_request_kwargs.yaml new file mode 100644 index 000000000000..49ce83b6ee43 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_avro_serializer_with_client_request_kwargs.yaml @@ -0,0 +1,34 @@ +interactions: +- request: + body: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' + headers: + Accept: + - application/json + Content-Length: + - '201' + Content-Type: + - application/json; serialization=Avro + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0 Python/3.9.0 (Windows-10-10.0.22000-SP0) + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemaGroups/fakegroup/schemas/example.avro.User?api-version=2021-10 + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 25 Feb 2022 23:54:21 GMT + location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 + schema-group-name: swathip-test-schema + schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 + schema-id-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/$schemas/08a5597bec1a47188aa3e0c99a69f5a3?api-version=2021-10 + schema-name: example.avro.User + schema-version: '1' + schema-versions-location: https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User/versions?api-version=2021-10 + server: Microsoft-HTTPAPI/2.0 + strict-transport-security: max-age=31536000 + status: + code: 204 + message: No Content + url: https://swathip-test-eventhubs.servicebus.windows.net/$schemaGroups/swathip-test-schema/schemas/example.avro.User?api-version=2021-10 +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml index 95e2a7d6ebca..3d8f3b665594 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_deserialize_readers_schema.yaml @@ -17,7 +17,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:09 GMT + date: Fri, 25 Feb 2022 23:54:22 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 @@ -45,7 +45,7 @@ interactions: string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' headers: content-type: application/json;serialization=Avro - date: Fri, 25 Feb 2022 00:51:09 GMT + date: Fri, 25 Feb 2022 23:54:22 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml index 480b6ff5d3eb..b7d04cfc6557 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml @@ -19,7 +19,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:10 GMT + date: Fri, 25 Feb 2022 23:54:24 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 @@ -53,7 +53,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:10 GMT + date: Fri, 25 Feb 2022 23:54:24 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 @@ -81,7 +81,7 @@ interactions: string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' headers: content-type: application/json;serialization=Avro - date: Fri, 25 Feb 2022 00:51:11 GMT + date: Fri, 25 Feb 2022 23:54:24 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml index 97fdeb34895b..1d231b18fdb5 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml @@ -19,7 +19,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:11 GMT + date: Fri, 25 Feb 2022 23:54:25 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 @@ -53,7 +53,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:12 GMT + date: Fri, 25 Feb 2022 23:54:26 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 @@ -81,7 +81,7 @@ interactions: string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' headers: content-type: application/json;serialization=Avro - date: Fri, 25 Feb 2022 00:51:12 GMT + date: Fri, 25 Feb 2022 23:54:26 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a5597bec1a47188aa3e0c99a69f5a3 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml index 8b4c7335af5d..361ab955507e 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_error_schema_as_record.yaml @@ -19,7 +19,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:14 GMT + date: Fri, 25 Feb 2022 23:54:28 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 8abba19fc05649a1a534166fc5060a08 @@ -47,7 +47,7 @@ interactions: string: '{"name":"User","namespace":"example.avro.error","type":"error","fields":[{"name":"name","type":"string"}]}' headers: content-type: application/json;serialization=Avro - date: Fri, 25 Feb 2022 00:51:14 GMT + date: Fri, 25 Feb 2022 23:54:28 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 8abba19fc05649a1a534166fc5060a08 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml index 9a2d9aa4ec58..359bcb3c195d 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_parse_record_name.yaml @@ -19,7 +19,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:15 GMT + date: Fri, 25 Feb 2022 23:54:29 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a3411941a0456cac738c425338b1db @@ -47,7 +47,7 @@ interactions: string: '{"namespace":"thrownaway","name":"User.avro","type":"record","fields":[{"name":"name","type":"string"}]}' headers: content-type: application/json;serialization=Avro - date: Fri, 25 Feb 2022 00:51:16 GMT + date: Fri, 25 Feb 2022 23:54:30 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 08a3411941a0456cac738c425338b1db @@ -81,7 +81,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:16 GMT + date: Fri, 25 Feb 2022 23:54:30 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: e088feb3752c492883a129cb7664daa6 @@ -109,7 +109,7 @@ interactions: string: '{"name":"User","type":"record","fields":[{"name":"name","type":"string"}]}' headers: content-type: application/json;serialization=Avro - date: Fri, 25 Feb 2022 00:51:17 GMT + date: Fri, 25 Feb 2022 23:54:31 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: e088feb3752c492883a129cb7664daa6 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml index f2cfcc29b03c..5f1fcb552b31 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_primitive.yaml @@ -17,7 +17,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:18 GMT + date: Fri, 25 Feb 2022 23:54:31 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/null/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: 3dedb9f5663943e4848dfa778c8ea9aa diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml index 0a3099c6b67b..97f7a6d99ff0 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer_async.test_serialize_record.yaml @@ -21,7 +21,7 @@ interactions: string: '' headers: content-length: '0' - date: Fri, 25 Feb 2022 00:51:20 GMT + date: Fri, 25 Feb 2022 23:54:33 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: f024ca12f4254593a89831cefc5c9e80 @@ -49,7 +49,7 @@ interactions: string: '{"name":"User","namespace":"example.avro.populatedrecord","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"},{"name":"married","type":"boolean"},{"name":"height","type":"float"},{"name":"randb","type":"bytes"}]}' headers: content-type: application/json;serialization=Avro - date: Fri, 25 Feb 2022 00:51:20 GMT + date: Fri, 25 Feb 2022 23:54:34 GMT location: https://fake_resource.servicebus.windows.net/:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/1?api-version=2021-10 schema-group-name: swathip-test-schema schema-id: f024ca12f4254593a89831cefc5c9e80 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py index 5542943a1106..4c06dbf18bb3 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -200,6 +200,27 @@ def test_basic_sr_avro_serializer_deserialize_readers_schema(self, schemaregistr with pytest.raises(SchemaDeserializationError): deserialized_data = sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_change_name) + @SchemaRegistryPowerShellPreparer() + def test_avro_serializer_with_client_request_kwargs(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, fully_qualified_namespace=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + schema = avro.schema.parse(schema_str) + + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + with pytest.raises(TypeError) as e: + serialized_metadata = sr_avro_serializer.serialize(dict_data, schema=schema_str, client_request_kwargs={"fake_kwarg": True}) + assert 'request() got an unexpected keyword' in str(e.value) + serialized_metadata = sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + with pytest.raises(TypeError) as e: + deserialized_data = sr_avro_serializer.deserialize(serialized_data_dict, client_request_kwargs={"fake_kwarg": True}) + assert 'request() got an unexpected keyword' in str(e.value) + ################################################################# ######################### PARSE SCHEMAS ######################### diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer_async.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer_async.py index e74eb98b8973..ac3af545bab8 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer_async.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer_async.py @@ -149,33 +149,56 @@ async def test_basic_sr_avro_serializer_deserialize_readers_schema(self, schemar schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - serialized_metadata = await sr_avro_serializer.serialize(dict_data, schema=schema_str) - content_type = serialized_metadata["content_type"] - serialized_data = serialized_metadata["data"] - - # readers_schema with removed field - readers_schema_remove_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]}]}""" - serialized_data_dict = {"data": serialized_data, "content_type": content_type} - deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_remove_field) - assert deserialized_data["name"] == u"Ben" - assert deserialized_data["favorite_number"] == 7 - - # readers_schema with extra field with default - readers_schema_extra_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}, {"name":"favorite_city","type":["string","null"], "default": "Redmond"}]}""" - serialized_data_dict = {"data": serialized_data, "content_type": content_type} - deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_extra_field) - assert deserialized_data["name"] == u"Ben" - assert deserialized_data["favorite_number"] == 7 - assert deserialized_data["favorite_color"] == "red" - assert deserialized_data["favorite_city"] == "Redmond" - - # readers_schema with changed name results in error - readers_schema_change_name = """{"namespace":"fakeexample.avro","type":"record","name":"fake_user","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" - with pytest.raises(SchemaDeserializationError): + async with sr_client, sr_avro_serializer: + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + serialized_metadata = await sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + + # readers_schema with removed field + readers_schema_remove_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]}]}""" + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_remove_field) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + + # readers_schema with extra field with default + readers_schema_extra_field = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}, {"name":"favorite_city","type":["string","null"], "default": "Redmond"}]}""" + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_extra_field) + assert deserialized_data["name"] == u"Ben" + assert deserialized_data["favorite_number"] == 7 + assert deserialized_data["favorite_color"] == "red" + assert deserialized_data["favorite_city"] == "Redmond" + + # readers_schema with changed name results in error + readers_schema_change_name = """{"namespace":"fakeexample.avro","type":"record","name":"fake_user","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + with pytest.raises(SchemaDeserializationError): + serialized_data_dict = {"data": serialized_data, "content_type": content_type} + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_change_name) + print(deserialized_data) + + @pytest.mark.asyncio + @SchemaRegistryPowerShellPreparer() + async def test_avro_serializer_with_client_request_kwargs(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_client(schemaregistry_fully_qualified_namespace) + sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" + + async with sr_client, sr_avro_serializer: + dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} + with pytest.raises(TypeError) as e: + serialized_metadata = await sr_avro_serializer.serialize(dict_data, schema=schema_str, client_request_kwargs={"fake_kwarg": True}) + assert 'request() got an unexpected keyword' in str(e.value) + serialized_metadata = await sr_avro_serializer.serialize(dict_data, schema=schema_str) + content_type = serialized_metadata["content_type"] + serialized_data = serialized_metadata["data"] + serialized_data_dict = {"data": serialized_data, "content_type": content_type} - deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, readers_schema=readers_schema_change_name) - print(deserialized_data) + with pytest.raises(TypeError) as e: + deserialized_data = await sr_avro_serializer.deserialize(serialized_data_dict, client_request_kwargs={"fake_kwarg": True}) + assert 'request() got an unexpected keyword' in str(e.value) #################################################################