From 00a3bbaedfb7f65a3a846cb73a02a49d8b3c0b1c Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Mon, 20 Sep 2021 14:51:35 -0700 Subject: [PATCH 1/7] make params required kwargs --- .../_schema_registry_avro_serializer.py | 34 +++++++------ .../azure/schemaregistry/_common/_schema.py | 50 ++++++++++--------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index e385630a81e2..456151834294 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -24,7 +24,7 @@ # # -------------------------------------------------------------------------- from io import BytesIO -from typing import Any, Dict, Union +from typing import Any, Dict, Union, Mapping import avro from ._constants import SCHEMA_ID_START_INDEX, SCHEMA_ID_LENGTH, DATA_START_INDEX @@ -36,20 +36,23 @@ class SchemaRegistryAvroSerializer(object): SchemaRegistryAvroSerializer provides the ability to serialize and deserialize data according to the given avro schema. It would automatically register, get and cache the schema. - :param client: The schema registry client + :keyword client: Required. The schema registry client which is used to register schema and retrieve schema from the service. - :type client: ~azure.schemaregistry.SchemaRegistryClient - :param str group_name: Schema group under which schema should be registered. + :paramtype client: ~azure.schemaregistry.SchemaRegistryClient + :keyword str group_name: Required. Schema group under which schema should be registered. :keyword bool auto_register_schemas: When true, register new schemas passed to serialize. Otherwise, and by default, fail if it has not been pre-registered in the registry. """ - def __init__(self, client, group_name, **kwargs): - # type: ("SchemaRegistryClient", str, Any) -> None - self._schema_group = group_name + def __init__(self, **kwargs): + # type: (Any) -> None + try: + self._schema_group = kwargs.pop("group_name") + self._schema_registry_client = kwargs.pop("client") # type: "SchemaRegistryClient" + except KeyError as e: + raise ValueError("'{}' is a required keyword.".format(e.args[0])) self._avro_serializer = AvroObjectSerializer(codec=kwargs.get("codec")) - self._schema_registry_client = client # type: "SchemaRegistryClient" self._auto_register_schemas = kwargs.get("auto_register_schemas", False) self._auto_register_schema_func = ( self._schema_registry_client.register_schema @@ -119,20 +122,23 @@ def _get_schema(self, schema_id, **kwargs): self._schema_to_id[schema_str] = schema_id return schema_str - def serialize(self, value, schema, **kwargs): - # type: (Dict[str, Any], Union[str, bytes], Any) -> bytes + def serialize(self, value, **kwargs): + # type: (Mapping[str, Any], Any) -> bytes """ Encode data with the given schema. The returns bytes are consisted of: The first 4 bytes denoting record format identifier. The following 32 bytes denoting schema id returned by schema registry service. The remaining bytes are the real data payload. :param value: The data to be encoded. - :type value: Dict[str, Any] - :param schema: The schema used to encode the data. - :type schema: str + :type value: Mapping[str, Any] + :keyword schema: Required. The schema used to encode the data. + :paramtype schema: str :rtype: bytes """ - raw_input_schema = schema + try: + raw_input_schema = kwargs.get("schema") + except KeyError as e: + raise ValueError("'{}' is a required keyword.".format(e.args[0])) try: cached_schema = self._user_input_schema_cache[raw_input_schema] except KeyError: diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index 582cb729f7b4..a83d1dc9aaa5 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -23,21 +23,19 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import Any, Optional +from typing import Any class SchemaProperties(object): """ Meta properties of a schema. - :ivar id: References specific schema in registry namespace. - :type id: str - :ivar location: URL location of schema, identified by schema group, schema name, and version. - :type location: str - :ivar serialization_type: Serialization type for the schema being stored. - :type serialization_type: str - :ivar version: Version of the returned schema. - :type version: int + :keyword id: Required. References specific schema in registry namespace. + :paramtype id: str + :keyword serialization_type: Required. Serialization type for the schema being stored. + :paramtype serialization_type: str + :keyword version: Required. Version of the returned schema. + :paramtype version: int .. admonition:: Example: @@ -52,24 +50,26 @@ class SchemaProperties(object): def __init__( self, - id=None, # pylint:disable=redefined-builtin **kwargs ): - # type: (Optional[str], Any) -> None - self.id = id - self.location = kwargs.get('location') - self.serialization_type = kwargs.get('serialization_type') - self.version = kwargs.get('version') + # type: (Any) -> None + try: + self.id = kwargs.pop('id') + self.location = kwargs.pop('location') + self.serialization_type = kwargs.pop('serialization_type') + self.version = kwargs.pop('version') + except KeyError as e: + raise ValueError("'{}' is a required keyword.".format(e.args[0])) class Schema(object): """ The schema content of a schema, along with id and meta properties. - :ivar content: The content of the schema. - :type content: str - :ivar properties: The properties of the schema. - :type properties: SchemaProperties + :keyword content: Required. The content of the schema. + :paramtype content: str + :keyword properties: Required. The properties of the schema. + :paramtype properties: SchemaProperties .. admonition:: Example: @@ -84,9 +84,11 @@ class Schema(object): def __init__( self, - content, - properties, + **kwargs ): - # type: (str, SchemaProperties) -> None - self.content = content - self.properties = properties + # type: (Any) -> None + try: + self.content = kwargs.pop("content") + self.properties = kwargs.pop("properties") + except KeyError as e: + raise ValueError("'{}' is a required keyword.".format(e.args[0])) From 8d3b87e64faea8fee10b98d3cef33b51669120ad Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Mon, 20 Sep 2021 15:14:02 -0700 Subject: [PATCH 2/7] update changelog --- .../azure-schemaregistry-avroserializer/CHANGELOG.md | 5 +++++ sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md index c0c620b6c1b5..1246832f4001 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md @@ -13,9 +13,14 @@ - `data` parameter in the `serialize` and `deserialize` methods on `SchemaRegistryAvroSerializer` has been renamed `value`. - `schema` parameter in the `serialize` method on `SchemaRegistryAvroSerializer` no longer accepts argument of type `bytes`. - `SchemaRegistryAvroSerializer` constructor no longer takes in the `codec` keyword argument. +- The following positional arguments are now required keyword arguments: + - `client` and `group_name` in `SchemaRegistryAvroSerializer` constructor + - `schema` in `serialize` on `SchemaRegistryAvroSerializer` ### Bugs Fixed +- `value` parameter in `serialize` on `SchemaRegistryAvroSerializer` takes type `Mapping` rather than `Dict`. + ### Other Changes ## 1.0.0b2 (2021-08-18) diff --git a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md index 8a35560abc10..7d8c5ffb0ea5 100644 --- a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md @@ -12,7 +12,11 @@ - `group_name`, which has been renamed from `schema_group` - `name`, which has been renamed from `schema_name` - `content`, which has been renamed from `schema_content` - - `serialization_type` + - `format`, which has been renamed from `serialization_type` +- `location` instance variable in `SchemaProperties` has been removed. +- The following positional arguments are now required keyword arguments: + - `id`, `serialization_type`, and `version` on `SchemaProperties` + - `content` and `properties` on `Schema` ### Bugs Fixed From 29d6eec15fcf84401f6ded159301751cd8fdda01 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Tue, 21 Sep 2021 18:19:24 -0700 Subject: [PATCH 3/7] adams comments --- .../CHANGELOG.md | 5 ++-- .../_schema_registry_avro_serializer.py | 4 ++-- .../azure-schemaregistry/CHANGELOG.md | 4 +--- .../azure/schemaregistry/_common/_schema.py | 24 +++++++++---------- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md index 1246832f4001..a24aa8e035c3 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features Added - `auto_register_schemas` keyword argument has been added to `SchemaRegistryAvroSerializer`, which will allow for automatically registering schemas passed in to the `serialize`. +- `value` parameter in `serialize` on `SchemaRegistryAvroSerializer` takes type `Mapping` rather than `Dict`. ### Breaking Changes @@ -15,12 +16,10 @@ - `SchemaRegistryAvroSerializer` constructor no longer takes in the `codec` keyword argument. - The following positional arguments are now required keyword arguments: - `client` and `group_name` in `SchemaRegistryAvroSerializer` constructor - - `schema` in `serialize` on `SchemaRegistryAvroSerializer` + - `schema` in `serialize` on `SchemaRegistryAvroSerializer` ### Bugs Fixed -- `value` parameter in `serialize` on `SchemaRegistryAvroSerializer` takes type `Mapping` rather than `Dict`. - ### Other Changes ## 1.0.0b2 (2021-08-18) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index 456151834294..e2490b3b25e0 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -51,7 +51,7 @@ def __init__(self, **kwargs): self._schema_group = kwargs.pop("group_name") self._schema_registry_client = kwargs.pop("client") # type: "SchemaRegistryClient" except KeyError as e: - raise ValueError("'{}' is a required keyword.".format(e.args[0])) + raise TypeError("'{}' is a required keyword.".format(e.args[0])) self._avro_serializer = AvroObjectSerializer(codec=kwargs.get("codec")) self._auto_register_schemas = kwargs.get("auto_register_schemas", False) self._auto_register_schema_func = ( @@ -138,7 +138,7 @@ def serialize(self, value, **kwargs): try: raw_input_schema = kwargs.get("schema") except KeyError as e: - raise ValueError("'{}' is a required keyword.".format(e.args[0])) + raise TypeError("'{}' is a required keyword.".format(e.args[0])) try: cached_schema = self._user_input_schema_cache[raw_input_schema] except KeyError: diff --git a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md index 7d8c5ffb0ea5..8b6aa13d1966 100644 --- a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md @@ -14,9 +14,7 @@ - `content`, which has been renamed from `schema_content` - `format`, which has been renamed from `serialization_type` - `location` instance variable in `SchemaProperties` has been removed. -- The following positional arguments are now required keyword arguments: - - `id`, `serialization_type`, and `version` on `SchemaProperties` - - `content` and `properties` on `Schema` +- `Schema` and `SchemaProperties` no longer have positional parameters, as they will not be constructed by the user. ### Bugs Fixed diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index a83d1dc9aaa5..42357f3c0f74 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -30,12 +30,12 @@ class SchemaProperties(object): """ Meta properties of a schema. - :keyword id: Required. References specific schema in registry namespace. - :paramtype id: str - :keyword serialization_type: Required. Serialization type for the schema being stored. - :paramtype serialization_type: str - :keyword version: Required. Version of the returned schema. - :paramtype version: int + :ivar id: References specific schema in registry namespace. + :type id: str + :ivar serialization_type: Serialization type for the schema being stored. + :type serialization_type: str + :ivar version: Version of the returned schema. + :type version: int .. admonition:: Example: @@ -59,17 +59,17 @@ def __init__( self.serialization_type = kwargs.pop('serialization_type') self.version = kwargs.pop('version') except KeyError as e: - raise ValueError("'{}' is a required keyword.".format(e.args[0])) + raise TypeError("'{}' is a required keyword.".format(e.args[0])) class Schema(object): """ The schema content of a schema, along with id and meta properties. - :keyword content: Required. The content of the schema. - :paramtype content: str - :keyword properties: Required. The properties of the schema. - :paramtype properties: SchemaProperties + :ivar content: The content of the schema. + :type content: str + :ivar properties: The properties of the schema. + :type properties: SchemaProperties .. admonition:: Example: @@ -91,4 +91,4 @@ def __init__( self.content = kwargs.pop("content") self.properties = kwargs.pop("properties") except KeyError as e: - raise ValueError("'{}' is a required keyword.".format(e.args[0])) + raise TypeError("'{}' is a required keyword.".format(e.args[0])) From e597cd5fdce7dd933181808e4dce5ad885027d57 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 24 Sep 2021 12:55:29 -0700 Subject: [PATCH 4/7] update avro tests + samples --- .../_schema_registry_avro_serializer.py | 2 +- .../samples/avro_serializer.py | 10 +++++----- .../samples/eventhub_receive_integration.py | 7 ++++--- .../samples/eventhub_send_integration.py | 7 ++++--- ...serializer_with_auto_register_schemas.yaml | 8 ++++---- ...ializer_without_auto_register_schemas.yaml | 8 ++++---- .../tests/schemaregistry_preparer.py | 8 ++++---- .../tests/test_avro_serializer.py | 20 ++++++++++--------- 8 files changed, 37 insertions(+), 33 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index e2490b3b25e0..a5f27b479948 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -136,7 +136,7 @@ def serialize(self, value, **kwargs): :rtype: bytes """ try: - raw_input_schema = kwargs.get("schema") + raw_input_schema = kwargs.pop("schema") except KeyError as e: raise TypeError("'{}' is a required keyword.".format(e.args[0])) try: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py index c7a40e56104e..592f1afa760f 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/avro_serializer.py @@ -33,7 +33,7 @@ CLIENT_ID=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_ID'] CLIENT_SECRET=os.environ['SCHEMA_REGISTRY_AZURE_CLIENT_SECRET'] -SCHEMA_REGISTRY_ENDPOINT=os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE=os.environ['SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE'] GROUP_NAME=os.environ['SCHEMA_REGISTRY_GROUP'] SCHEMA_STRING = """ {"namespace": "example.avro", @@ -59,9 +59,9 @@ def serialize(serializer): dict_data_alice = {"name": u"Alice", "favorite_number": 15, "favorite_color": u"green"} # Schema would be automatically registered into Schema Registry and cached locally. - payload_ben = serializer.serialize(dict_data_ben, SCHEMA_STRING) + payload_ben = serializer.serialize(dict_data_ben, schema=SCHEMA_STRING) # The second call won't trigger a service call. - payload_alice = serializer.serialize(dict_data_alice, SCHEMA_STRING) + payload_alice = serializer.serialize(dict_data_alice, schema=SCHEMA_STRING) print('Encoded bytes are: ', payload_ben) print('Encoded bytes are: ', payload_alice) @@ -79,8 +79,8 @@ def deserialize(serializer, bytes_payload): if __name__ == '__main__': - schema_registry = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_ENDPOINT, credential=token_credential) - serializer = SchemaRegistryAvroSerializer(schema_registry, GROUP_NAME) + schema_registry = SchemaRegistryClient(endpoint=SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE, credential=token_credential) + serializer = SchemaRegistryAvroSerializer(client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True) bytes_data_ben, bytes_data_alice = serialize(serializer) dict_data_ben = deserialize(serializer, bytes_data_ben) dict_data_alice = deserialize(serializer, bytes_data_alice) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_receive_integration.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_receive_integration.py index 783fc7167603..8ed76259298b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_receive_integration.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_receive_integration.py @@ -19,7 +19,7 @@ EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] -SCHEMA_REGISTRY_ENDPOINT = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE'] GROUP_NAME = os.environ['SCHEMA_REGISTRY_GROUP'] @@ -47,10 +47,11 @@ def on_event(partition_context, event): # create a SchemaRegistryAvroSerializer instance avro_serializer = SchemaRegistryAvroSerializer( client=SchemaRegistryClient( - endpoint=SCHEMA_REGISTRY_ENDPOINT, + fully_qualified_namespace=SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE, credential=DefaultAzureCredential() ), - group_name=GROUP_NAME + group_name=GROUP_NAME, + auto_register_schemas=True ) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_send_integration.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_send_integration.py index 8169e9ea2734..cb0e49102248 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_send_integration.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_send_integration.py @@ -20,7 +20,7 @@ EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR'] EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] -SCHEMA_REGISTRY_ENDPOINT = os.environ['SCHEMA_REGISTRY_ENDPOINT'] +SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE'] GROUP_NAME = os.environ['SCHEMA_REGISTRY_GROUP'] SCHEMA_STRING = """ @@ -61,10 +61,11 @@ def send_event_data_batch(producer, serializer): # create a SchemaRegistryAvroSerializer instance avro_serializer = SchemaRegistryAvroSerializer( client=SchemaRegistryClient( - endpoint=SCHEMA_REGISTRY_ENDPOINT, + fully_qualified_namespace=SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE, credential=DefaultAzureCredential() ), - group_name=GROUP_NAME + group_name=GROUP_NAME, + auto_register_schemas=True ) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml index 2410144713ec..138b96d57d44 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml @@ -23,12 +23,12 @@ interactions: uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/example.avro.User?api-version=2017-04 response: body: - string: '{"id":"041afcdb34a546faa3aa26a991567e32"}' + string: '{"id":"fc61e4d3e31b46f6a758fa1b67f35cc5"}' headers: content-type: - application/json date: - - Wed, 08 Sep 2021 22:17:05 GMT + - Fri, 24 Sep 2021 19:54:45 GMT location: - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2017-04 server: @@ -38,9 +38,9 @@ interactions: transfer-encoding: - chunked x-schema-id: - - 041afcdb34a546faa3aa26a991567e32 + - fc61e4d3e31b46f6a758fa1b67f35cc5 x-schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/041afcdb34a546faa3aa26a991567e32?api-version=2017-04 + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/fc61e4d3e31b46f6a758fa1b67f35cc5?api-version=2017-04 x-schema-type: - Avro x-schema-version: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml index bad3c1562982..2d3f07b5e19d 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml @@ -23,12 +23,12 @@ interactions: uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/example.avro.User?api-version=2017-04 response: body: - string: '{"id":"041afcdb34a546faa3aa26a991567e32"}' + string: '{"id":"fc61e4d3e31b46f6a758fa1b67f35cc5"}' headers: content-type: - application/json date: - - Wed, 08 Sep 2021 22:17:06 GMT + - Fri, 24 Sep 2021 19:54:47 GMT location: - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2017-04 server: @@ -38,9 +38,9 @@ interactions: transfer-encoding: - chunked x-schema-id: - - 041afcdb34a546faa3aa26a991567e32 + - fc61e4d3e31b46f6a758fa1b67f35cc5 x-schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/041afcdb34a546faa3aa26a991567e32?api-version=2017-04 + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/fc61e4d3e31b46f6a758fa1b67f35cc5?api-version=2017-04 x-schema-type: - Avro x-schema-version: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py index f1f78608d0d0..bbf139cbac38 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/schemaregistry_preparer.py @@ -30,9 +30,9 @@ ) -SCHEMA_REGISTRY_ENDPOINT_PARAM = "schemaregistry_endpoint" +SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE_PARAM = "schemaregistry_fully_qualified_namespace" SCHEMA_REGISTRY_GROUP_PARAM = "schemaregistry_group" -SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME = 'SCHEMA_REGISTRY_ENDPOINT' +SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE_ENV_KEY_NAME = 'SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE' SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME = 'SCHEMA_REGISTRY_GROUP' @@ -59,13 +59,13 @@ def create_resource(self, name, **kwargs): # TODO: right now the endpoint/group is fixed, as there is no way to create/delete resources using api, in the future we should be able to dynamically create and remove resources if self.is_live: return { - SCHEMA_REGISTRY_ENDPOINT_PARAM: os.environ[SCHEMA_REGISTRY_ENDPOINT_ENV_KEY_NAME], + SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE_PARAM: os.environ[SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE_ENV_KEY_NAME], SCHEMA_REGISTRY_GROUP_PARAM: os.environ[SCHEMA_REGISTRY_GROUP_ENV_KEY_NAME] } else: return { - SCHEMA_REGISTRY_ENDPOINT_PARAM: "sr-playground.servicebus.windows.net", + SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE_PARAM: "sr-playground.servicebus.windows.net", SCHEMA_REGISTRY_GROUP_PARAM: "azsdk_python_test_group" } diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py index 9e8e6308c052..3d7b7f56608a 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -32,7 +32,7 @@ from devtools_testutils import AzureTestCase, PowerShellPreparer -SchemaRegistryPowerShellPreparer = functools.partial(PowerShellPreparer, "schemaregistry", schemaregistry_endpoint="fake_resource.servicebus.windows.net/", schemaregistry_group="fakegroup") +SchemaRegistryPowerShellPreparer = functools.partial(PowerShellPreparer, "schemaregistry", schemaregistry_fully_qualified_namespace="fake_resource.servicebus.windows.net/", schemaregistry_group="fakegroup") class SchemaRegistryAvroSerializerTests(AzureTestCase): @@ -75,15 +75,16 @@ def test_raw_avro_serializer_negative(self): raw_avro_object_serializer.serialize(dict_data_missing_required_field, schema) @SchemaRegistryPowerShellPreparer() - def test_basic_sr_avro_serializer_with_auto_register_schemas(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - sr_avro_serializer = SchemaRegistryAvroSerializer(sr_client, schemaregistry_group, auto_register_schemas=True) + def test_basic_sr_avro_serializer_with_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + # TODO: AFTER RELEASING azure-schemaregistry=1.0.0b3, UPDATE 'endpoint' to 'fully_qualified_namespace' + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - encoded_data = sr_avro_serializer.serialize(dict_data, schema_str) + encoded_data = sr_avro_serializer.serialize(dict_data, schema=schema_str) assert schema_str in sr_avro_serializer._user_input_schema_cache assert str(avro.schema.parse(schema_str)) in sr_avro_serializer._schema_to_id @@ -102,15 +103,16 @@ def test_basic_sr_avro_serializer_with_auto_register_schemas(self, schemaregistr sr_avro_serializer.close() @SchemaRegistryPowerShellPreparer() - def test_basic_sr_avro_serializer_without_auto_register_schemas(self, schemaregistry_endpoint, schemaregistry_group, **kwargs): - sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_endpoint) - sr_avro_serializer = SchemaRegistryAvroSerializer(sr_client, schemaregistry_group) + def test_basic_sr_avro_serializer_without_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + # TODO: AFTER RELEASING azure-schemaregistry=1.0.0b3, UPDATE 'endpoint' to 'fully_qualified_namespace' + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group) schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema = avro.schema.parse(schema_str) dict_data = {"name": u"Ben", "favorite_number": 7, "favorite_color": u"red"} - encoded_data = sr_avro_serializer.serialize(dict_data, schema_str) + encoded_data = sr_avro_serializer.serialize(dict_data, schema=schema_str) assert schema_str in sr_avro_serializer._user_input_schema_cache assert str(avro.schema.parse(schema_str)) in sr_avro_serializer._schema_to_id From d9ea79b48bc4c8144b36844d6779902148e5724a Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 24 Sep 2021 12:58:35 -0700 Subject: [PATCH 5/7] fix samples --- .../samples/eventhub_receive_integration.py | 3 ++- .../samples/eventhub_send_integration.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_receive_integration.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_receive_integration.py index 8ed76259298b..2cf4d0095b28 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_receive_integration.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_receive_integration.py @@ -45,9 +45,10 @@ def on_event(partition_context, event): # create a SchemaRegistryAvroSerializer instance +# TODO: after 'azure-schemaregistry==1.0.0b3' is released, update 'endpoint' to 'fully_qualified_namespace' avro_serializer = SchemaRegistryAvroSerializer( client=SchemaRegistryClient( - fully_qualified_namespace=SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE, + endpoint=SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE, credential=DefaultAzureCredential() ), group_name=GROUP_NAME, diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_send_integration.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_send_integration.py index cb0e49102248..2638b849cf8d 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_send_integration.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples/eventhub_send_integration.py @@ -59,9 +59,10 @@ def send_event_data_batch(producer, serializer): # create a SchemaRegistryAvroSerializer instance +# TODO: after 'azure-schemaregistry==1.0.0b3' is released, update 'endpoint' to 'fully_qualified_namespace' avro_serializer = SchemaRegistryAvroSerializer( client=SchemaRegistryClient( - fully_qualified_namespace=SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE, + endpoint=SCHEMA_REGISTRY_FULLY_QUALIFIED_NAMESPACE, credential=DefaultAzureCredential() ), group_name=GROUP_NAME, From a66b44061c0bb8898655a57f735bbb097bc2d58f Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 24 Sep 2021 13:48:31 -0700 Subject: [PATCH 6/7] pylint error --- .../avroserializer/_schema_registry_avro_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index a5f27b479948..d322730bfd16 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -24,7 +24,7 @@ # # -------------------------------------------------------------------------- from io import BytesIO -from typing import Any, Dict, Union, Mapping +from typing import Any, Dict, Mapping import avro from ._constants import SCHEMA_ID_START_INDEX, SCHEMA_ID_LENGTH, DATA_START_INDEX From 3c8be95dea675d52ed93679694d58563769dbc7e Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 24 Sep 2021 16:46:58 -0700 Subject: [PATCH 7/7] adams comments --- .../azure/schemaregistry/_common/_schema.py | 2 +- ...est_schema_registry.test_schema_basic.yaml | 24 +++++++++---------- ...gistry.test_schema_negative_no_schema.yaml | 12 +++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index 530c4871488d..cd706e3329e0 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -54,7 +54,7 @@ def __init__( ): # type: (Any) -> None self.id = kwargs.pop('id') - self.format = kwargs.get('format') + self.format = kwargs.pop('format') self.version = kwargs.pop('version') diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml index 643693326529..01adea1ae411 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_basic.yaml @@ -20,18 +20,18 @@ interactions: uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/test-schema-basic31c70f88?api-version=2020-09-01-preview response: body: - string: '{"id":"c91dd34b22d143ab85321c871ca60cec"}' + string: '{"id":"c9e4d79c4a06400aac374b9711bf98f8"}' headers: content-type: - application/json date: - - Fri, 24 Sep 2021 19:10:17 GMT + - Fri, 24 Sep 2021 23:46:00 GMT location: - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/test-schema-basic31c70f88/versions/1?api-version=2020-09-01-preview schema-id: - - c91dd34b22d143ab85321c871ca60cec + - c9e4d79c4a06400aac374b9711bf98f8 schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/c91dd34b22d143ab85321c871ca60cec?api-version=2020-09-01-preview + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/c9e4d79c4a06400aac374b9711bf98f8?api-version=2020-09-01-preview schema-version: - '1' schema-versions-location: @@ -59,7 +59,7 @@ interactions: User-Agent: - azsdk-python-azureschemaregistry/1.0.0b3 Python/3.9.0 (Windows-10-10.0.19041-SP0) method: GET - uri: https://fake_resource.servicebus.windows.net/$schemagroups/getSchemaById/c91dd34b22d143ab85321c871ca60cec?api-version=2020-09-01-preview + uri: https://fake_resource.servicebus.windows.net/$schemagroups/getSchemaById/c9e4d79c4a06400aac374b9711bf98f8?api-version=2020-09-01-preview response: body: string: '{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}' @@ -67,13 +67,13 @@ interactions: content-type: - application/json date: - - Fri, 24 Sep 2021 19:10:18 GMT + - Fri, 24 Sep 2021 23:46:00 GMT location: - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/test-schema-basic31c70f88/versions/1?api-version=2020-09-01-preview schema-id: - - c91dd34b22d143ab85321c871ca60cec + - c9e4d79c4a06400aac374b9711bf98f8 schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/c91dd34b22d143ab85321c871ca60cec?api-version=2020-09-01-preview + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/c9e4d79c4a06400aac374b9711bf98f8?api-version=2020-09-01-preview schema-version: - '1' schema-versions-location: @@ -110,18 +110,18 @@ interactions: uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/test-schema-basic31c70f88?api-version=2020-09-01-preview response: body: - string: '{"id":"c91dd34b22d143ab85321c871ca60cec"}' + string: '{"id":"c9e4d79c4a06400aac374b9711bf98f8"}' headers: content-type: - application/json date: - - Fri, 24 Sep 2021 19:10:18 GMT + - Fri, 24 Sep 2021 23:46:01 GMT location: - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/test-schema-basic31c70f88/versions/1?api-version=2020-09-01-preview schema-id: - - c91dd34b22d143ab85321c871ca60cec + - c9e4d79c4a06400aac374b9711bf98f8 schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/c91dd34b22d143ab85321c871ca60cec?api-version=2020-09-01-preview + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/c9e4d79c4a06400aac374b9711bf98f8?api-version=2020-09-01-preview schema-version: - '1' schema-versions-location: diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml index 77dbe11ce957..d3b4e6df3427 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml +++ b/sdk/schemaregistry/azure-schemaregistry/tests/recordings/test_schema_registry.test_schema_negative_no_schema.yaml @@ -15,14 +15,14 @@ interactions: response: body: string: '{"Code":400,"Detail":"SubCode=40000, UnknownType:The request is invalid. - [MGResponseHttpError=BadRequest]. TrackingId:95d455ee-9f97-4982-95eb-164e5823f8dd_G7, + [MGResponseHttpError=BadRequest]. TrackingId:21ad828b-f18a-41e9-b6b2-a7999b16edf0_G16, SystemTracker:swathip-test-eventhubs.servicebus.windows.net:$schemagroups\/getSchemaById\/a, - Timestamp:2021-09-24T19:10:20"}' + Timestamp:2021-09-24T23:46:03"}' headers: content-type: - application/json date: - - Fri, 24 Sep 2021 19:10:19 GMT + - Fri, 24 Sep 2021 23:46:02 GMT server: - Microsoft-HTTPAPI/2.0 strict-transport-security: @@ -48,13 +48,13 @@ interactions: response: body: string: '{"Code":404,"Detail":"Schema id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa does - not exist. TrackingId:b14d75ed-2e3e-41b0-860c-461dd6bb4270_G7, SystemTracker:swathip-test-eventhubs.servicebus.windows.net:$schemagroups\/getSchemaById\/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, - Timestamp:2021-09-24T19:10:20"}' + not exist. TrackingId:84fba0d2-8892-49c4-83f7-a46bb0c51a72_G16, SystemTracker:swathip-test-eventhubs.servicebus.windows.net:$schemagroups\/getSchemaById\/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, + Timestamp:2021-09-24T23:46:03"}' headers: content-type: - application/json date: - - Fri, 24 Sep 2021 19:10:19 GMT + - Fri, 24 Sep 2021 23:46:03 GMT server: - Microsoft-HTTPAPI/2.0 strict-transport-security: