From f1530dc2efe29866efbb1bb4775437bb874ddd0e Mon Sep 17 00:00:00 2001 From: Joel Hendrix Date: Thu, 29 Feb 2024 09:25:31 -0800 Subject: [PATCH] Updates for aztables metadata (#22472) Include returned metadata in more APIs. Make the amount of returned metadata configurable. --- sdk/data/aztables/CHANGELOG.md | 6 ++- sdk/data/aztables/assets.json | 2 +- sdk/data/aztables/client.go | 22 ++++++-- sdk/data/aztables/client_test.go | 67 ++++++++++++++++++++++-- sdk/data/aztables/constants.go | 10 ++++ sdk/data/aztables/internal/version.go | 2 +- sdk/data/aztables/models.go | 3 ++ sdk/data/aztables/options.go | 25 +++++---- sdk/data/aztables/responses.go | 6 ++- sdk/data/aztables/service_client.go | 21 +++++++- sdk/data/aztables/service_client_test.go | 34 ++++++++++-- 11 files changed, 169 insertions(+), 29 deletions(-) diff --git a/sdk/data/aztables/CHANGELOG.md b/sdk/data/aztables/CHANGELOG.md index 8ed21c3216ec..44276d10008f 100644 --- a/sdk/data/aztables/CHANGELOG.md +++ b/sdk/data/aztables/CHANGELOG.md @@ -1,8 +1,12 @@ # Release History -## 1.1.1 (Unreleased) +## 1.2.0 (Unreleased) ### Features Added +* Methods `Client.AddEntity` and `ServiceClient.NewListTablesPager` now include OData metadata in their responses. +* The amount of OData metadata returned has been made configurable for the following methods: + * `Client.AddEntity`, `Client.GetEntity`, `Client.NewListEntitiesPager`, and `ServiceClient.NewListTablesPager`. + * Use one of the following constants to specify the amount: `MetadataFormatFull`, `MetadataFormatMinimal`, or `MetadataFormatNone`. ### Breaking Changes diff --git a/sdk/data/aztables/assets.json b/sdk/data/aztables/assets.json index 60251978c5e3..2ca6056dc7e5 100644 --- a/sdk/data/aztables/assets.json +++ b/sdk/data/aztables/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/data/aztables", - "Tag": "go/data/aztables_45893b48dc" + "Tag": "go/data/aztables_ec73894009" } diff --git a/sdk/data/aztables/client.go b/sdk/data/aztables/client.go index e1d934990ad6..9a0a5f4febec 100644 --- a/sdk/data/aztables/client.go +++ b/sdk/data/aztables/client.go @@ -13,7 +13,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" generated "github.com/Azure/azure-sdk-for-go/sdk/data/aztables/internal" ) @@ -220,8 +219,9 @@ func (t *Client) GetEntity(ctx context.Context, partitionKey string, rowKey stri options = &GetEntityOptions{} } - genOptions, queryOptions := options.toGenerated() - resp, err := t.client.QueryEntityWithPartitionAndRowKey(ctx, t.name, prepareKey(partitionKey), prepareKey(rowKey), genOptions, queryOptions) + resp, err := t.client.QueryEntityWithPartitionAndRowKey(ctx, t.name, prepareKey(partitionKey), prepareKey(rowKey), nil, &generated.QueryOptions{ + Format: options.Format, + }) if err != nil { return GetEntityResponse{}, err } @@ -254,18 +254,30 @@ func (t *Client) AddEntity(ctx context.Context, entity []byte, options *AddEntit if err != nil { return AddEntityResponse{}, err } - resp, err := t.client.InsertEntity(ctx, t.name, &generated.TableClientInsertEntityOptions{TableEntityProperties: mapEntity, ResponsePreference: to.Ptr(generated.ResponseFormatReturnNoContent)}, nil) + + if options == nil { + options = &AddEntityOptions{} + } + + resp, err := t.client.InsertEntity(ctx, t.name, &generated.TableClientInsertEntityOptions{TableEntityProperties: mapEntity}, &generated.QueryOptions{ + Format: options.Format, + }) if err != nil { err = checkEntityForPkRk(&mapEntity, err) return AddEntityResponse{}, err } + marshalledValue, err := json.Marshal(resp.Value) + if err != nil { + return AddEntityResponse{}, err + } var ETag azcore.ETag if resp.ETag != nil { ETag = azcore.ETag(*resp.ETag) } return AddEntityResponse{ - ETag: ETag, + ETag: ETag, + Value: marshalledValue, }, nil } diff --git a/sdk/data/aztables/client_test.go b/sdk/data/aztables/client_test.go index 10b589849d1e..1c2e03145aa0 100644 --- a/sdk/data/aztables/client_test.go +++ b/sdk/data/aztables/client_test.go @@ -54,6 +54,11 @@ func TestCreateTable(t *testing.T) { } } +type mdforAddGet struct { + Metadata string `json:"odata.metadata"` + Type string `json:"odata.type"` // only for full metadata +} + func TestAddEntity(t *testing.T) { for _, service := range services { t.Run(fmt.Sprintf("%v_%v", t.Name(), service), func(t *testing.T) { @@ -66,8 +71,13 @@ func TestAddEntity(t *testing.T) { marshalledEntity, err := json.Marshal(simpleEntity) require.NoError(t, err) - _, err = client.AddEntity(ctx, marshalledEntity, nil) + resp, err := client.AddEntity(ctx, marshalledEntity, nil) require.NoError(t, err) + require.NotEmpty(t, resp.Value) + var md mdforAddGet + require.NoError(t, json.Unmarshal(resp.Value, &md)) + require.NotEmpty(t, md.Metadata) + require.Empty(t, md.Type) }) } } @@ -83,8 +93,18 @@ func TestAddComplexEntity(t *testing.T) { marshalledEntity, err := json.Marshal(entity) require.NoError(t, err) - _, err = client.AddEntity(ctx, marshalledEntity, nil) + resp, err := client.AddEntity(ctx, marshalledEntity, &AddEntityOptions{ + Format: to.Ptr(MetadataFormatFull), + }) require.NoError(t, err) + require.NotEmpty(t, resp.Value) + var md mdforAddGet + require.NoError(t, json.Unmarshal(resp.Value, &md)) + require.NotEmpty(t, md.Metadata) + if service == "storage" { + // cosmos doesn't send full metadata + require.NotEmpty(t, md.Type) + } }) } } @@ -165,6 +185,11 @@ func TestMergeEntity(t *testing.T) { preMerge, err := client.GetEntity(ctx, entityToCreate.PartitionKey, entityToCreate.RowKey, nil) require.NoError(t, err) + require.NotEmpty(t, preMerge.Value) + var md mdforAddGet + require.NoError(t, json.Unmarshal(preMerge.Value, &md)) + require.NotEmpty(t, md.Metadata) + require.Empty(t, md.Type) var unMarshalledPreMerge map[string]any err = json.Unmarshal(preMerge.Value, &unMarshalledPreMerge) @@ -245,8 +270,18 @@ func TestInsertEntity(t *testing.T) { list := &ListEntitiesOptions{Filter: &filter} // 2. Query for basic Entity - preMerge, err := client.GetEntity(ctx, entityToCreate.PartitionKey, entityToCreate.RowKey, nil) + preMerge, err := client.GetEntity(ctx, entityToCreate.PartitionKey, entityToCreate.RowKey, &GetEntityOptions{ + Format: to.Ptr(MetadataFormatFull), + }) require.NoError(t, err) + require.NotEmpty(t, preMerge.Value) + var md mdforAddGet + require.NoError(t, json.Unmarshal(preMerge.Value, &md)) + require.NotEmpty(t, md.Metadata) + if service == "storage" { + // cosmos doesn't send full metadata + require.NotEmpty(t, md.Type) + } var unMarshalledPreMerge map[string]any err = json.Unmarshal(preMerge.Value, &unMarshalledPreMerge) @@ -304,6 +339,11 @@ func TestInsertEntityTwice(t *testing.T) { } } +type mdForListEntities struct { + Timestamp time.Time `json:"Timestamp"` + ID string `json:"odata.id"` // only for full metadata +} + func TestQuerySimpleEntity(t *testing.T) { for _, service := range services { t.Run(fmt.Sprintf("%v_%v", t.Name(), service), func(t *testing.T) { @@ -328,7 +368,8 @@ func TestQuerySimpleEntity(t *testing.T) { var resp ListEntitiesResponse pager := client.NewListEntitiesPager(list) for pager.More() { - resp, err := pager.NextPage(ctx) + var err error + resp, err = pager.NextPage(ctx) require.NoError(t, err) require.Equal(t, len(resp.Entities), expectedCount) } @@ -353,6 +394,11 @@ func TestQuerySimpleEntity(t *testing.T) { require.Equal(t, b.String, (*entitiesToCreate)[i].String) require.Equal(t, b.Integer, (*entitiesToCreate)[i].Integer) require.Equal(t, b.Bool, (*entitiesToCreate)[i].Bool) + + var md mdForListEntities + require.NoError(t, json.Unmarshal(e, &md)) + require.False(t, md.Timestamp.IsZero()) + require.Empty(t, md.ID) } }) } @@ -375,7 +421,10 @@ func TestQueryComplexEntity(t *testing.T) { filter := "RowKey lt '5'" expectedCount := 4 - options := &ListEntitiesOptions{Filter: &filter} + options := &ListEntitiesOptions{ + Filter: &filter, + Format: to.Ptr(MetadataFormatFull), + } pager := client.NewListEntitiesPager(options) for pager.More() { @@ -396,6 +445,14 @@ func TestQueryComplexEntity(t *testing.T) { require.Equal(t, model.Float, (entitiesToCreate)[idx].Float) require.Equal(t, model.DateTime, (entitiesToCreate)[idx].DateTime) require.Equal(t, model.Byte, (entitiesToCreate)[idx].Byte) + + var md mdForListEntities + require.NoError(t, json.Unmarshal(entity, &md)) + require.False(t, md.Timestamp.IsZero()) + if service == "storage" { + // cosmos doesn't send full metadata + require.NotEmpty(t, md.ID) + } } } }) diff --git a/sdk/data/aztables/constants.go b/sdk/data/aztables/constants.go index ef6b96b01c0d..09601548614c 100644 --- a/sdk/data/aztables/constants.go +++ b/sdk/data/aztables/constants.go @@ -56,6 +56,16 @@ func toGeneratedStatusType(g *generated.GeoReplicationStatusType) *GeoReplicatio return nil } +// MetadataFormat specifies the level of OData metadata returned with an entity. +// https://learn.microsoft.com/rest/api/storageservices/payload-format-for-table-service-operations#json-format-applicationjson-versions-2013-08-15-and-later +type MetadataFormat = generated.ODataMetadataFormat + +const ( + MetadataFormatFull MetadataFormat = generated.ODataMetadataFormatApplicationJSONODataFullmetadata + MetadataFormatMinimal MetadataFormat = generated.ODataMetadataFormatApplicationJSONODataMinimalmetadata + MetadataFormatNone MetadataFormat = generated.ODataMetadataFormatApplicationJSONODataNometadata +) + // SASProtocol indicates the SAS protocol type SASProtocol string diff --git a/sdk/data/aztables/internal/version.go b/sdk/data/aztables/internal/version.go index 37ce6e7cb1c7..dd9c04d3ffd2 100644 --- a/sdk/data/aztables/internal/version.go +++ b/sdk/data/aztables/internal/version.go @@ -8,5 +8,5 @@ package internal const ( ModuleName = "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" - Version = "v1.1.1" + Version = "v1.2.0" ) diff --git a/sdk/data/aztables/models.go b/sdk/data/aztables/models.go index 7e53ac2a00ec..71ee4fde274f 100644 --- a/sdk/data/aztables/models.go +++ b/sdk/data/aztables/models.go @@ -105,6 +105,9 @@ func (t *ServiceProperties) toGenerated() *generated.TableServiceProperties { type TableProperties struct { // The name of the table. Name *string `json:"TableName,omitempty"` + + // The OData properties of the table in JSON format. + Value []byte } // RetentionPolicy - The retention policy. diff --git a/sdk/data/aztables/options.go b/sdk/data/aztables/options.go index fc5c88e05710..a5364f98241e 100644 --- a/sdk/data/aztables/options.go +++ b/sdk/data/aztables/options.go @@ -5,13 +5,14 @@ package aztables import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" generated "github.com/Azure/azure-sdk-for-go/sdk/data/aztables/internal" ) // AddEntityOptions contains optional parameters for Client.AddEntity type AddEntityOptions struct { - // placeholder for future optional parameters + // Format specifies the amount of metadata returned. + // The default is MetadataFormatMinimal. + Format *MetadataFormat } // CreateTableOptions contains optional parameters for Client.Create and ServiceClient.CreateTable @@ -52,11 +53,9 @@ func (g *GetAccessPolicyOptions) toGenerated() *generated.TableClientGetAccessPo // GetEntityOptions contains optional parameters for Client.GetEntity type GetEntityOptions struct { - // placeholder for future optional parameters -} - -func (g *GetEntityOptions) toGenerated() (*generated.TableClientQueryEntityWithPartitionAndRowKeyOptions, *generated.QueryOptions) { - return &generated.TableClientQueryEntityWithPartitionAndRowKeyOptions{}, &generated.QueryOptions{Format: to.Ptr(generated.ODataMetadataFormatApplicationJSONODataMinimalmetadata)} + // Format specifies the amount of metadata returned. + // The default is MetadataFormatMinimal. + Format *MetadataFormat } // GetPropertiesOptions contains optional parameters for Client.GetProperties @@ -94,6 +93,10 @@ type ListEntitiesOptions struct { // The NextRowKey to start paging from NextRowKey *string + + // Format specifies the amount of metadata returned. + // The default is MetadataFormatMinimal. + Format *MetadataFormat } func (l *ListEntitiesOptions) toQueryOptions() *generated.QueryOptions { @@ -103,7 +106,7 @@ func (l *ListEntitiesOptions) toQueryOptions() *generated.QueryOptions { return &generated.QueryOptions{ Filter: l.Filter, - Format: to.Ptr(generated.ODataMetadataFormatApplicationJSONODataMinimalmetadata), + Format: l.Format, Select: l.Select, Top: l.Top, } @@ -122,6 +125,10 @@ type ListTablesOptions struct { // NextTableName is the continuation token for the next table to page from NextTableName *string + + // Format specifies the amount of metadata returned. + // The default is MetadataFormatMinimal. + Format *MetadataFormat } func (l *ListTablesOptions) toQueryOptions() *generated.QueryOptions { @@ -131,7 +138,7 @@ func (l *ListTablesOptions) toQueryOptions() *generated.QueryOptions { return &generated.QueryOptions{ Filter: l.Filter, - Format: to.Ptr(generated.ODataMetadataFormatApplicationJSONODataMinimalmetadata), + Format: l.Format, Select: l.Select, Top: l.Top, } diff --git a/sdk/data/aztables/responses.go b/sdk/data/aztables/responses.go index 5f2b56895832..b9886c29c79d 100644 --- a/sdk/data/aztables/responses.go +++ b/sdk/data/aztables/responses.go @@ -7,7 +7,11 @@ import "github.com/Azure/azure-sdk-for-go/sdk/azcore" // AddEntityResponse contains response fields for Client.AddEntityResponse type AddEntityResponse struct { + // ETag contains the information returned from the ETag header response. ETag azcore.ETag + + // The OData properties of the table entity in JSON format. + Value []byte } // CreateTableResponse contains response fields for Client.Create and ServiceClient.CreateTable @@ -36,7 +40,7 @@ type GetEntityResponse struct { // ETag contains the information returned from the ETag header response. ETag azcore.ETag - // The properties of the table entity. + // The OData properties of the table entity in JSON format. Value []byte } diff --git a/sdk/data/aztables/service_client.go b/sdk/data/aztables/service_client.go index 999a518d1bac..4b22a85f2a31 100644 --- a/sdk/data/aztables/service_client.go +++ b/sdk/data/aztables/service_client.go @@ -5,6 +5,7 @@ package aztables import ( "context" + "encoding/json" "errors" "strings" "time" @@ -162,8 +163,26 @@ func (t *ServiceClient) NewListTablesPager(listOptions *ListTablesOptions) *runt tableProps := make([]*TableProperties, len(resp.Value)) for i := range resp.Value { + odataValues := map[string]any{} + if resp.Value[i].ODataEditLink != nil { + odataValues["odata.editLink"] = *resp.Value[i].ODataEditLink + } + if resp.Value[i].ODataID != nil { + odataValues["odata.id"] = *resp.Value[i].ODataID + } + if resp.Value[i].ODataType != nil { + odataValues["odata.type"] = *resp.Value[i].ODataType + } + var odataJSON []byte + if len(odataValues) > 0 { + odataJSON, err = json.Marshal(odataValues) + if err != nil { + return ListTablesResponse{}, err + } + } tableProps[i] = &TableProperties{ - Name: resp.Value[i].TableName, + Name: resp.Value[i].TableName, + Value: odataJSON, } } diff --git a/sdk/data/aztables/service_client_test.go b/sdk/data/aztables/service_client_test.go index 32d456357d89..3436f0ac4e89 100644 --- a/sdk/data/aztables/service_client_test.go +++ b/sdk/data/aztables/service_client_test.go @@ -4,6 +4,7 @@ package aztables import ( + "encoding/json" "fmt" "testing" "time" @@ -119,6 +120,9 @@ func TestQueryTable(t *testing.T) { require.LessOrEqual(t, len(resp.Tables), 2) resultCount += len(resp.Tables) pageCount++ + for _, table := range resp.Tables { + require.Nil(t, table.Value) + } } require.Equal(t, resultCount, tableCount-1) @@ -129,37 +133,57 @@ func TestQueryTable(t *testing.T) { } } +type mdForListTables struct { + EditLink string `json:"odata.editLink"` + ID string `json:"odata.id"` + Type string `json:"odata.type"` +} + func TestListTables(t *testing.T) { for _, service := range services { t.Run(fmt.Sprintf("%v_%v", t.Name(), service), func(t *testing.T) { - service, delete := initServiceTest(t, service, NewSpanValidator(t, SpanMatcher{ + client, delete := initServiceTest(t, service, NewSpanValidator(t, SpanMatcher{ Name: "Pager[ListTablesResponse].NextPage", })) defer delete() tableName, err := createRandomName(t, tableNamePrefix) require.NoError(t, err) - err = clearAllTables(service) + err = clearAllTables(client) require.NoError(t, err) for i := 0; i < 5; i++ { - _, err := service.CreateTable(ctx, fmt.Sprintf("%v%v", tableName, i), nil) + _, err := client.CreateTable(ctx, fmt.Sprintf("%v%v", tableName, i), nil) require.NoError(t, err) } count := 0 - pager := service.NewListTablesPager(nil) + pager := client.NewListTablesPager(&ListTablesOptions{ + Format: to.Ptr(MetadataFormatFull), + }) for pager.More() { resp, err := pager.NextPage(ctx) require.NoError(t, err) count += len(resp.Tables) + + for _, table := range resp.Tables { + if service == "storage" { + // cosmos doesn't send full metadata + require.NotEmpty(t, table.Value) + var md mdForListTables + require.NoError(t, json.Unmarshal(table.Value, &md)) + require.NotEmpty(t, md.EditLink) + require.NotEmpty(t, md.ID) + require.NotEmpty(t, md.Type) + } + } } require.Equal(t, 5, count) deleteTable := func() { for i := 0; i < 5; i++ { - _, err := service.DeleteTable(ctx, fmt.Sprintf("%v%v", tableName, i), nil) + _, err := client.DeleteTable(ctx, fmt.Sprintf("%v%v", tableName, i), nil) if err != nil { fmt.Printf("Error cleaning up test. %v\n", err.Error()) }