From fc9c511b68d27eac78fac65aa5886d8c7cab4560 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 23 Sep 2020 16:54:18 -0700 Subject: [PATCH] adds customization to disable s3 auto decompress gzip --- .../aws/go/codegen/AwsGoDependency.java | 2 +- .../customization/AwsCustomGoDependency.java | 1 + .../DynamoDBValidateResponseChecksum.java | 4 +- .../customization/S3AcceptEncodingGzip.java | 98 ++++++++ ...mithy.go.codegen.integration.GoIntegration | 1 + .../accept-encoding/accept_encoding_gzip.go | 168 ++++++++++++++ .../accept_encoding_gzip_test.go | 215 ++++++++++++++++++ service/internal/accept-encoding/go.mod | 5 + 8 files changed, 491 insertions(+), 3 deletions(-) create mode 100644 codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java create mode 100644 service/internal/accept-encoding/accept_encoding_gzip.go create mode 100644 service/internal/accept-encoding/accept_encoding_gzip_test.go create mode 100644 service/internal/accept-encoding/go.mod diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsGoDependency.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsGoDependency.java index a0d926a18c4..f5f34abbc09 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsGoDependency.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsGoDependency.java @@ -51,6 +51,6 @@ protected static GoDependency aws(String relativePath, String alias) { } private static final class Versions { - private static final String AWS_SDK = "v0.0.0-20200923000934-8cf2e0ac6dea"; + private static final String AWS_SDK = "v0.0.0-20200923180406-b8bee42bd556"; } } diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java index b076416c2c5..9c19d028786 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java @@ -25,6 +25,7 @@ public final class AwsCustomGoDependency extends AwsGoDependency { public static final GoDependency DYNAMODB_CUSTOMIZATION = aws("service/dynamodb/internal/customizations", "ddbcust"); public static final GoDependency S3_CUSTOMIZATION = aws("service/s3/internal/customizations", "s3cust"); public static final GoDependency APIGATEWAY_CUSTOMIZATION = aws("service/apigateway/internal/customizations", "agcust"); + public static final GoDependency ACCEPT_ENCODING_CUSTOMIZATION = aws("service/internal/accept-encoding", "acceptencodingcust"); private AwsCustomGoDependency() { super(); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java index 4720a449b55..32f3b2caacc 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java @@ -82,9 +82,9 @@ private void writeMiddlewareHelper(GoWriter writer) { writer.openBlock("func $L(stack *middleware.Stack, options Options) {", "}", GZIP_ADDER, () -> { writer.write("$T(stack, $T{Enable: options.$L})", SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER, - AwsCustomGoDependency.DYNAMODB_CUSTOMIZATION).build(), + AwsCustomGoDependency.ACCEPT_ENCODING_CUSTOMIZATION).build(), SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER + "Options", - AwsCustomGoDependency.DYNAMODB_CUSTOMIZATION).build(), + AwsCustomGoDependency.ACCEPT_ENCODING_CUSTOMIZATION).build(), GZIP_CLIENT_OPTION ); }); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java new file mode 100644 index 00000000000..a792e651c00 --- /dev/null +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java @@ -0,0 +1,98 @@ +package software.amazon.smithy.aws.go.codegen.customization; + +import java.util.List; +import java.util.logging.Logger; +import software.amazon.smithy.aws.go.codegen.AddAwsConfigFields; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.go.codegen.GoDelegator; +import software.amazon.smithy.go.codegen.GoSettings; +import software.amazon.smithy.go.codegen.GoWriter; +import software.amazon.smithy.go.codegen.SymbolUtils; +import software.amazon.smithy.go.codegen.integration.ConfigField; +import software.amazon.smithy.go.codegen.integration.GoIntegration; +import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; +import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.utils.ListUtils; + +/** + * S3AcceptEncodingGzip adds a customization for s3 client to disable + * auto decoding of GZip content by Golang HTTP Client. + * + * This customization provides an option on the S3 client options to enable + * AcceptEncoding for GZIP. The flag if set, will enable auto decompression of + * GZIP by the S3 Client. + * + * By default, the client's auto decompression of GZIP content is turned off. + */ +public class S3AcceptEncodingGzip implements GoIntegration { + private static final Logger LOGGER = Logger.getLogger(AddAwsConfigFields.class.getName()); + + private static final String GZIP_DISABLE = "disableAcceptEncodingGzip"; + private static final String GZIP_INTERNAL_ADDER = "AddAcceptEncodingGzip"; + + /** + * Gets the sort order of the customization from -128 to 127, with lowest + * executed first. + * + * @return Returns the sort order, defaults to -50. + */ + @Override + public byte getOrder() { + return 127; + } + + @Override + public void writeAdditionalFiles( + GoSettings settings, + Model model, + SymbolProvider symbolProvider, + GoDelegator goDelegator + ) { + if (!isServiceS3(model, settings.getService(model))) { + return; + } + + goDelegator.useShapeWriter(settings.getService(model), this::writeMiddlewareHelper); + } + + private void writeMiddlewareHelper(GoWriter writer) { + writer.openBlock("func $L(stack *middleware.Stack) {", "}", GZIP_DISABLE, () -> { + writer.write("$T(stack, $T{})", + SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER, + AwsCustomGoDependency.ACCEPT_ENCODING_CUSTOMIZATION).build(), + SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER + "Options", + AwsCustomGoDependency.ACCEPT_ENCODING_CUSTOMIZATION).build() + ); + }); + writer.insertTrailingNewline(); + } + + @Override + public List getClientPlugins() { + return ListUtils.of( + // register disableAcceptEncodingGzip middleware + RuntimeClientPlugin.builder() + .servicePredicate(S3AcceptEncodingGzip::isServiceS3) + .registerMiddleware(MiddlewareRegistrar.builder() + .resolvedFunction(SymbolUtils.createValueSymbolBuilder(GZIP_DISABLE) + .build()) + .build() + ) + .build() + ); + } + + /** + * Return true if service is S3. + * + * @param model the model used for generation. + * @param service the service shape for which default HTTP Client is generated. + * @return true if service is S3 + */ + private static boolean isServiceS3(Model model, ServiceShape service) { + return service.expectTrait(ServiceTrait.class).getSdkId().equalsIgnoreCase("S3"); + } +} diff --git a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index 0b220cc896e..18a27daac0b 100644 --- a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -11,3 +11,4 @@ software.amazon.smithy.aws.go.codegen.customization.DynamoDBValidateResponseChec software.amazon.smithy.aws.go.codegen.customization.S3UpdateEndpoint software.amazon.smithy.aws.go.codegen.customization.APIGatewayAcceptHeader software.amazon.smithy.aws.go.codegen.customization.BackfillOptionalAuthTrait +software.amazon.smithy.aws.go.codegen.customization.S3AcceptEncodingGzip diff --git a/service/internal/accept-encoding/accept_encoding_gzip.go b/service/internal/accept-encoding/accept_encoding_gzip.go new file mode 100644 index 00000000000..2785d2d1f9f --- /dev/null +++ b/service/internal/accept-encoding/accept_encoding_gzip.go @@ -0,0 +1,168 @@ +package accept_encoding + +import ( + "compress/gzip" + "context" + "fmt" + "io" + + "github.com/awslabs/smithy-go" + "github.com/awslabs/smithy-go/middleware" + smithyhttp "github.com/awslabs/smithy-go/transport/http" +) + +const acceptEncodingHeaderKey = "Accept-Encoding" +const contentEncodingHeaderKey = "Content-Encoding" + +// AddAcceptEncodingGzipOptions provides the options for the +// AddAcceptEncodingGzip middleware setup. +type AddAcceptEncodingGzipOptions struct { + Enable bool +} + +// AddAcceptEncodingGzip explicitly adds handling for accept-encoding GZIP +// middleware to the operation stack. This allows checksums to be correctly +// computed without disabling GZIP support. +func AddAcceptEncodingGzip(stack *middleware.Stack, options AddAcceptEncodingGzipOptions) { + if options.Enable { + stack.Finalize.Add(&AcceptEncodingGzipMiddleware{}, middleware.Before) + stack.Deserialize.Insert(&DecompressGzipMiddleware{}, "OperationDeserializer", middleware.After) + return + } + + stack.Finalize.Add(&DisableAcceptEncodingGzipMiddleware{}, middleware.Before) +} + +// DisableAcceptEncodingGzipMiddleware provides the middleware that will +// disable the underlying http client automatically enabling for gzip +// decompress content-encoding support. +type DisableAcceptEncodingGzipMiddleware struct{} + +// ID returns the id for the middleware. +func (*DisableAcceptEncodingGzipMiddleware) ID() string { + return "DisableAcceptEncodingGzipMiddleware" +} + +// HandleFinalize implements the FinalizeMiddlware interface. +func (*DisableAcceptEncodingGzipMiddleware) HandleFinalize( + ctx context.Context, input middleware.FinalizeInput, next middleware.FinalizeHandler, +) ( + output middleware.FinalizeOutput, metadata middleware.Metadata, err error, +) { + req, ok := input.Request.(*smithyhttp.Request) + if !ok { + return output, metadata, &smithy.SerializationError{ + Err: fmt.Errorf("unknown request type %T", input.Request), + } + } + + // Explicitly enable gzip support, this will prevent the http client from + // auto extracting the zipped content. + req.Header.Set(acceptEncodingHeaderKey, "identity") + + return next.HandleFinalize(ctx, input) +} + +// AcceptEncodingGzipMiddleware provides a middleware to enable support for +// gzip responses, with manual decompression. This prevents the underlying HTTP +// client from performing the gzip decompression automatically. +type AcceptEncodingGzipMiddleware struct{} + +// ID returns the id for the middleware. +func (*AcceptEncodingGzipMiddleware) ID() string { return "AcceptEncodingGzipMiddleware" } + +// HandleFinalize implements the FinalizeMiddlware interface. +func (*AcceptEncodingGzipMiddleware) HandleFinalize( + ctx context.Context, input middleware.FinalizeInput, next middleware.FinalizeHandler, +) ( + output middleware.FinalizeOutput, metadata middleware.Metadata, err error, +) { + req, ok := input.Request.(*smithyhttp.Request) + if !ok { + return output, metadata, &smithy.SerializationError{ + Err: fmt.Errorf("unknown request type %T", input.Request), + } + } + + // Explicitly enable gzip support, this will prevent the http client from + // auto extracting the zipped content. + req.Header.Set(acceptEncodingHeaderKey, "gzip") + + return next.HandleFinalize(ctx, input) +} + +// DecompressGzipMiddleware provides the middleware for decompressing a gzip +// response from the service. +type DecompressGzipMiddleware struct{} + +// ID returns the id for the middleware. +func (*DecompressGzipMiddleware) ID() string { return "DecompressGzipMiddleware" } + +// HandleDeserialize implements the DeserializeMiddlware interface. +func (*DecompressGzipMiddleware) HandleDeserialize( + ctx context.Context, input middleware.DeserializeInput, next middleware.DeserializeHandler, +) ( + output middleware.DeserializeOutput, metadata middleware.Metadata, err error, +) { + output, metadata, err = next.HandleDeserialize(ctx, input) + if err != nil { + return output, metadata, err + } + + resp, ok := output.RawResponse.(*smithyhttp.Response) + if !ok { + return output, metadata, &smithy.DeserializationError{ + Err: fmt.Errorf("unknown response type %T", output.RawResponse), + } + } + if v := resp.Header.Get(contentEncodingHeaderKey); v != "gzip" { + return output, metadata, err + } + + // Clear content length since it will no longer be valid once the response + // body is decompressed. + resp.Header.Del("Content-Length") + resp.ContentLength = -1 + + resp.Body = wrapGzipReader(resp.Body) + + return output, metadata, err +} + +type gzipReader struct { + reader io.ReadCloser + gzip *gzip.Reader +} + +func wrapGzipReader(reader io.ReadCloser) *gzipReader { + return &gzipReader{ + reader: reader, + } +} + +// Read wraps the gzip reader around the underlying io.Reader to extract the +// response bytes on the fly. +func (g *gzipReader) Read(b []byte) (n int, err error) { + if g.gzip == nil { + g.gzip, err = gzip.NewReader(g.reader) + if err != nil { + g.gzip = nil // ensure uninitialized gzip value isn't used in close. + return 0, fmt.Errorf("failed to decompress gzip response, %w", err) + } + } + + return g.gzip.Read(b) +} + +func (g *gzipReader) Close() error { + if g.gzip == nil { + return nil + } + + if err := g.gzip.Close(); err != nil { + g.reader.Close() + return fmt.Errorf("failed to decompress gzip response, %w", err) + } + + return g.reader.Close() +} diff --git a/service/internal/accept-encoding/accept_encoding_gzip_test.go b/service/internal/accept-encoding/accept_encoding_gzip_test.go new file mode 100644 index 00000000000..8c381a06a88 --- /dev/null +++ b/service/internal/accept-encoding/accept_encoding_gzip_test.go @@ -0,0 +1,215 @@ +package accept_encoding + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/hex" + "io" + "io/ioutil" + "net/http" + "testing" + + "github.com/awslabs/smithy-go/middleware" + smithyhttp "github.com/awslabs/smithy-go/transport/http" +) + +func TestAddAcceptEncodingGzip(t *testing.T) { + cases := map[string]struct { + Enable bool + }{ + "disabled": { + Enable: false, + }, + "enabled": { + Enable: true, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + stack := middleware.NewStack("test", smithyhttp.NewStackRequest) + + stack.Deserialize.Add(&stubOpDeserializer{}, middleware.After) + + AddAcceptEncodingGzip(stack, AddAcceptEncodingGzipOptions{ + Enable: c.Enable, + }) + + id := "OperationDeserializer" + if m, ok := stack.Deserialize.Get(id); !ok || m == nil { + t.Fatalf("expect %s not to be removed", id) + } + + if c.Enable { + id = (*AcceptEncodingGzipMiddleware)(nil).ID() + if m, ok := stack.Finalize.Get(id); !ok || m == nil { + t.Fatalf("expect %s to be present.", id) + } + + id = (*DecompressGzipMiddleware)(nil).ID() + if m, ok := stack.Deserialize.Get(id); !ok || m == nil { + t.Fatalf("expect %s to be present.", id) + } + return + } + id = (*AcceptEncodingGzipMiddleware)(nil).ID() + if m, ok := stack.Finalize.Get(id); ok || m != nil { + t.Fatalf("expect %s not to be present.", id) + } + + id = (*DecompressGzipMiddleware)(nil).ID() + if m, ok := stack.Deserialize.Get(id); ok || m != nil { + t.Fatalf("expect %s not to be present.", id) + } + }) + } +} + +func TestAcceptEncodingGzipMiddleware(t *testing.T) { + m := &AcceptEncodingGzipMiddleware{} + + _, _, err := m.HandleFinalize(context.Background(), + middleware.FinalizeInput{ + Request: smithyhttp.NewStackRequest(), + }, + middleware.FinalizeHandlerFunc( + func(ctx context.Context, input middleware.FinalizeInput) ( + output middleware.FinalizeOutput, metadata middleware.Metadata, err error, + ) { + req, ok := input.Request.(*smithyhttp.Request) + if !ok || req == nil { + t.Fatalf("expect smithy request, got %T", input.Request) + } + + actual := req.Header.Get(acceptEncodingHeaderKey) + if e, a := "gzip", actual; e != a { + t.Errorf("expect %v accept-encoding, got %v", e, a) + } + + return output, metadata, err + }), + ) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } +} + +func TestDecompressGzipMiddleware(t *testing.T) { + cases := map[string]struct { + Response *smithyhttp.Response + ExpectBody []byte + ExpectContentLength int64 + }{ + "not compressed": { + Response: &smithyhttp.Response{ + Response: &http.Response{ + StatusCode: 200, + Header: http.Header{}, + ContentLength: 2, + Body: &wasClosedReadCloser{ + Reader: bytes.NewBuffer([]byte(`{}`)), + }, + }, + }, + ExpectBody: []byte(`{}`), + ExpectContentLength: 2, + }, + "compressed": { + Response: &smithyhttp.Response{ + Response: &http.Response{ + StatusCode: 200, + Header: http.Header{ + contentEncodingHeaderKey: []string{"gzip"}, + }, + ContentLength: 10, + Body: func() io.ReadCloser { + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + w.Write([]byte(`{}`)) + w.Close() + + return &wasClosedReadCloser{Reader: &buf} + }(), + }, + }, + ExpectBody: []byte(`{}`), + ExpectContentLength: -1, // Length empty because was decompressed + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + m := &DecompressGzipMiddleware{} + + var origRespBody io.Reader + output, _, err := m.HandleDeserialize(context.Background(), + middleware.DeserializeInput{}, + middleware.DeserializeHandlerFunc( + func(ctx context.Context, input middleware.DeserializeInput) ( + output middleware.DeserializeOutput, metadata middleware.Metadata, err error, + ) { + output.RawResponse = c.Response + origRespBody = c.Response.Body + return output, metadata, err + }), + ) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + + resp, ok := output.RawResponse.(*smithyhttp.Response) + if !ok || resp == nil { + t.Fatalf("expect smithy request, got %T", output.RawResponse) + } + + if e, a := c.ExpectContentLength, resp.ContentLength; e != a { + t.Errorf("expect %v content-length, got %v", e, a) + } + + actual, err := ioutil.ReadAll(resp.Body) + if e, a := c.ExpectBody, actual; !bytes.Equal(e, a) { + t.Errorf("expect body equal\nexpect:\n%s\nactual:\n%s", + hex.Dump(e), hex.Dump(a)) + } + + if err := resp.Body.Close(); err != nil { + t.Fatalf("expect no close error, got %v", err) + } + + if c, ok := origRespBody.(interface{ WasClosed() bool }); ok { + if !c.WasClosed() { + t.Errorf("expect original reader closed, but was not") + } + } + }) + } +} + +type stubOpDeserializer struct{} + +func (*stubOpDeserializer) ID() string { return "OperationDeserializer" } +func (*stubOpDeserializer) HandleDeserialize( + ctx context.Context, input middleware.DeserializeInput, next middleware.DeserializeHandler, +) ( + output middleware.DeserializeOutput, metadata middleware.Metadata, err error, +) { + return next.HandleDeserialize(ctx, input) +} + +type wasClosedReadCloser struct { + io.Reader + closed bool +} + +func (c *wasClosedReadCloser) WasClosed() bool { + return c.closed +} + +func (c *wasClosedReadCloser) Close() error { + c.closed = true + if v, ok := c.Reader.(io.Closer); ok { + return v.Close() + } + return nil +} diff --git a/service/internal/accept-encoding/go.mod b/service/internal/accept-encoding/go.mod new file mode 100644 index 00000000000..ec492f9dd96 --- /dev/null +++ b/service/internal/accept-encoding/go.mod @@ -0,0 +1,5 @@ +module github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding + +go 1.15 + +require github.com/awslabs/smithy-go v0.0.0-20200923230457-84341f7d6722