From 9a86474b7c00b320301ab746e5d697cf6f0134fb Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 11 Nov 2020 01:44:01 -0800 Subject: [PATCH 01/24] java codegen changes to generate presign clients for each valid service, presign operation for valid operations --- .../AwsHttpPresignURLClientGenerator.java | 556 ++++++++++++------ 1 file changed, 386 insertions(+), 170 deletions(-) diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java index 5dc35c03bd5..f0513f12f41 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java @@ -17,123 +17,257 @@ package software.amazon.smithy.aws.go.codegen; -import java.util.ArrayList; -import java.util.List; +import java.util.Map; +import java.util.Set; import software.amazon.smithy.aws.go.codegen.customization.AwsCustomGoDependency; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait; +import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait; import software.amazon.smithy.codegen.core.Symbol; import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.go.codegen.GoDelegator; +import software.amazon.smithy.go.codegen.GoSettings; import software.amazon.smithy.go.codegen.GoWriter; import software.amazon.smithy.go.codegen.OperationGenerator; import software.amazon.smithy.go.codegen.SmithyGoDependency; import software.amazon.smithy.go.codegen.SymbolUtils; -import software.amazon.smithy.go.codegen.integration.ProtocolUtils; +import software.amazon.smithy.go.codegen.integration.GoIntegration; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.model.shapes.Shape; -import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.utils.MapUtils; +import software.amazon.smithy.utils.SetUtils; -public class AwsHttpPresignURLClientGenerator { +public class AwsHttpPresignURLClientGenerator implements GoIntegration { + // constants private static final String CONVERT_TO_PRESIGN_MIDDLEWARE_NAME = "convertToPresignMiddleware"; - private final Model model; - private final SymbolProvider symbolProvider; + private static final String PRESIGN_CLIENT = "PresignClient"; + private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true); - private final Symbol presignClientSymbol; - private final Symbol newPresignClientSymbol; + private static final String NEW_CLIENT = "NewPresignClient"; + private static final String NEW_CLIENT_FROM_SERVICE = "NewPresignClientWrapper"; + private static final String NEW_CLIENT_FROM_CONFIG = "NewPresignClientFromConfig"; - private final OperationShape operation; - private final Symbol operationSymbol; - private final Shape operationInput; - private final Symbol operationInputSymbol; + private static final String PRESIGN_OPTIONS = "PresignOptions"; + private static final Symbol presignOptionsSymbol = buildSymbol(PRESIGN_OPTIONS, true); - private final boolean exported; + private static final String PRESIGN_OPTIONS_FROM_CLIENT_OPTIONS = "WithPresignClientFromClientOptions"; + private static final String PRESIGN_OPTIONS_FROM_EXPIRES = "WithPresignExpires"; - private final List convertToPresignMiddlewareHelpers; + private static final String PRESIGN_SIGNER = "Presigner"; + private static final Symbol presignerInterfaceSymbol = SymbolUtils.createPointableSymbolBuilder( + "HTTPPresigner", AwsGoDependency.AWS_SIGNER_V4 + ).build(); + private static final Symbol v4NewPresignerSymbol = SymbolUtils.createPointableSymbolBuilder( + "NewSigner", AwsGoDependency.AWS_SIGNER_V4 + ).build(); - private AwsHttpPresignURLClientGenerator(Builder builder) { - this.exported = builder.exported; + // map of service to list of operations for which presignedURL client and operation should + // be generated. + private static final Map> PRESIGNER_MAP = MapUtils.of( + ShapeId.from("com.amazonaws.s3#AmazonS3"), SetUtils.of( + ShapeId.from("com.amazonaws.s3#PutObject")), - this.model = SmithyBuilder.requiredState("model", builder.model); - this.symbolProvider = SmithyBuilder.requiredState("symbolProvider", builder.symbolProvider); - this.convertToPresignMiddlewareHelpers = builder.convertToPresignMiddlewareHelpers; + ShapeId.from("com.amazonaws.rds#AmazonRDSv19"), SetUtils.of( + ShapeId.from("com.amazonaws.rds#CopyDBSnapshot"), + ShapeId.from("com.amazonaws.rds#CreateDBInstanceReadReplica"), + ShapeId.from("com.amazonaws.rds#CopyDBClusterSnapshot"), + ShapeId.from("com.amazonaws.rds#CreateDBCluster")), - this.operation = SmithyBuilder.requiredState("operation", builder.operation); - this.operationSymbol = symbolProvider.toSymbol(operation); + ShapeId.from("com.amazonaws.ec2#AmazonEC2"), SetUtils.of( + ShapeId.from("com.amazonaws.ec2#CopySnapshot")) - this.operationInput = ProtocolUtils.expectInput(model, operation); - this.operationInputSymbol = symbolProvider.toSymbol(operationInput); + // TODO other services + ); - this.presignClientSymbol = buildPresignClientSymbol(operationSymbol, exported); - this.newPresignClientSymbol = buildNewPresignClientSymbol(operationSymbol, exported); + // build pointable symbols + private static Symbol buildSymbol(String name, boolean exported) { + if (!exported) { + name = Character.toLowerCase(name.charAt(0)) + name.substring(1); + } + return SymbolUtils.createPointableSymbolBuilder(name). + build(); } /** - * Writes the Presign client's type and methods. + * generates code to iterate thru func optionals and assign value into the dest variable * - * @param writer writer to write to + * @param writer GoWriter to write the code to + * @param src variable name that denotes functional options + * @param dest variable in which result of processed functional options are stored + * @param destType value type used by functional options */ - public void writePresignClientType(GoWriter writer) { - writer.addUseImports(SmithyGoDependency.CONTEXT); - writer.addUseImports(AwsGoDependency.AWS_SIGNER_V4); + private static final void processFunctionalOptions( + GoWriter writer, + String src, + String dest, + Symbol destType + ) { + writer.write("var $L $T", dest, destType); + writer.openBlock("for _, fn := range $L {", "}", src, () -> { + writer.write("fn(&$L)", dest); + }).insertTrailingNewline(); + } - writer.openBlock("type $T struct {", "}", presignClientSymbol, () -> { - writer.write("client *Client"); - writer.write("presigner *v4.Signer"); + /** + * variables needed in scope: + * * client + * * presignOptions + *

+ * generates code to assign client, presigner and return a new presign client + * + * @param writer the writer to write to + */ + private final void returnPresignClientConstructor( + GoWriter writer, + Model model, + ServiceShape serviceShape + ) { + writer.write("var presigner $T", presignerInterfaceSymbol); + writer.openBlock("if presignOptions.Presigner != nil {", "} else {", () -> { + writer.write("presigner = presignOptions.Presigner"); + }).write("presigner = $T() }", v4NewPresignerSymbol).insertTrailingNewline(); + + writer.openBlock("return &$L{", "}", presignClientSymbol, () -> { + writer.write("client: client, presigner: presigner,"); + // if s3 assign expires value on client + if (isS3ServiceShape(model, serviceShape)) { + writer.write("expires: presignOptions.Expires,"); + } }); - writer.openBlock("func $L(options Options, optFns ...func(*Options)) *$T {", "}", - newPresignClientSymbol.getName(), - presignClientSymbol, - () -> { - writer.openBlock("return &$T{", "}", presignClientSymbol, () -> { - writer.write("client: New(options, optFns...),"); + } - writer.addUseImports(AwsGoDependency.AWS_SIGNER_V4); - writer.write("presigner: v4.NewSigner(),"); - }); - }); + @Override + public byte getOrder() { + // The associated customization ordering is relative to operation deserializers + // and thus the integration should be added at the end. + return 127; + } - writer.addUseImports(SmithyGoDependency.NET_HTTP); - writer.openBlock( - // TODO presign with expire can be supported with a builder param that adds an additional expires param to presign signature. + @Override + public void writeAdditionalFiles( + GoSettings settings, + Model model, + SymbolProvider symbolProvider, + GoDelegator goDelegator + ) { + ServiceShape serviceShape = settings.getService(model); + + if (!PRESIGNER_MAP.containsKey(serviceShape.getId())) { + return; + } - // TODO Should this return a v4.PresignedHTTPRequest type instead of individual fields? - "func (c *$T) Presign$T(ctx context.Context, params $P, optFns ...func(*Options)) " - + "(string, http.Header, error) {", + Set validOperations = PRESIGNER_MAP.get(serviceShape.getId()); + if (validOperations.isEmpty()) { + return; + } + + // delegator for service shape + goDelegator.useShapeWriter(serviceShape, (writer) -> { + // generate presign options and helpers per service + writePresignOptionType(writer, model, symbolProvider, serviceShape); + + // generate Presign client per service + writePresignClientType(writer, model, symbolProvider, serviceShape); + + // generate client helpers such as copyAPIClient, GetAPIClientOptions() + writePresignClientHelpers(writer, model, symbolProvider, serviceShape); + + // generate convertToPresignMiddleware per service + writeConvertToPresignMiddleware(writer, model, symbolProvider, serviceShape); + }); + + + for (ShapeId operationId : serviceShape.getAllOperations()) { + OperationShape operationShape = model.expectShape(operationId, OperationShape.class); + if (!validOperations.contains(operationShape.getId())) { + continue; + } + + goDelegator.useShapeWriter(operationShape, (writer) -> { + // generate presign operation function for a client operation. + writePresignOperationFunction(writer, model, symbolProvider, serviceShape, operationShape); + }); + } + } + + private void writePresignOperationFunction( + GoWriter writer, + Model model, + SymbolProvider symbolProvider, + ServiceShape serviceShape, + OperationShape operationShape + ) { + Symbol operationSymbol = symbolProvider.toSymbol(operationShape); + + Shape operationInputShape = model.expectShape(operationShape.getInput().get()); + Symbol operationInputSymbol = symbolProvider.toSymbol(operationInputShape); + + writer.openBlock( + "func (c *$T) Presign$T(ctx context.Context, params $P, optFns ...func($P)) " + + "(req *v4.PresignedHTTPRequest, err error) {", "}", - presignClientSymbol, operationSymbol, operationInputSymbol, + presignClientSymbol, operationSymbol, operationInputSymbol, presignOptionsSymbol, () -> { Symbol nopClient = SymbolUtils.createPointableSymbolBuilder("NopClient", SmithyGoDependency.SMITHY_HTTP_TRANSPORT) .build(); - writer.write("if params == nil { params = &$T{} }", operationInputSymbol); - writer.write(""); + writer.write("if params == nil { params = &$T{} }", operationInputSymbol).insertTrailingNewline(); + + // process presignerOptions + processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); + + // check if presigner was set for presignerOptions + writer.openBlock("if presignOptions.Presigner != nil {", "}", () -> { + writer.write( + "c = NewPresignClientWrapper(c.client, func (o $P) { o.Presigner = presignOptions.Presigner })", + presignOptionsSymbol); + }); + + writer.write("clientOptFns := presignOptions.ClientOptions"); - // TODO could be replaced with a `WithAPIOptions` client option helper. - // TODO could be replaced with a `WithHTTPClient` client option helper. - writer.openBlock("optFns = append(optFns, func(o *Options) {", "})", () -> { + writer.openBlock("clientOptFns = append(clientOptFns, func(o *Options) {", "})", () -> { writer.write("o.HTTPClient = &$T{}", nopClient); }); - writer.write(""); + writer.insertTrailingNewline(); Symbol withIsPresigning = SymbolUtils.createValueSymbolBuilder("WithIsPresigning", AwsCustomGoDependency.PRESIGNEDURL_CUSTOMIZATION).build(); writer.write("ctx = $T(ctx)", withIsPresigning); - writer.openBlock("result, _, err := c.client.invokeOperation(ctx, $S, params, optFns,", ")", + writer.openBlock("result, _, err := c.client.invokeOperation(ctx, $S, params, clientOptFns,", ")", operationSymbol.getName(), () -> { writer.write("$L,", OperationGenerator .getAddOperationMiddlewareFuncName(operationSymbol)); writer.write("c.$L,", CONVERT_TO_PRESIGN_MIDDLEWARE_NAME); }); - writer.write("if err != nil { return ``, nil, err }"); - writer.write(""); + writer.write("if err != nil { return req, err }"); + writer.insertTrailingNewline(); writer.write("out := result.(*v4.PresignedHTTPRequest)"); - writer.write("return out.URL, out.SignedHeader, nil"); + writer.write("return out, nil"); }); + } + /** + * generates a helper to mutate request middleware stack in favor of generating a presign URL request + * + * @param writer the writer to write to + * @param model the service model + * @param symbolProvider the symbol provider + * @param serviceShape the service for which helper is generated + */ + private void writeConvertToPresignMiddleware( + GoWriter writer, + Model model, + SymbolProvider symbolProvider, + ServiceShape serviceShape + ) { Symbol smithyStack = SymbolUtils.createPointableSymbolBuilder("Stack", SmithyGoDependency.SMITHY_MIDDLEWARE) .build(); @@ -166,138 +300,220 @@ public void writePresignClientType(GoWriter writer) { presignMiddleware, smithyAfter); writer.write("if err != nil { return err }"); - convertToPresignMiddlewareHelpers.forEach((symbol) -> { - writer.write("err = $T(stack)", symbol); + // if protocol used is ec2query or query + if (serviceShape.hasTrait(AwsQueryTrait.ID) || serviceShape.hasTrait(Ec2QueryTrait.ID)) { + // presigned url should convert to Get request + Symbol queryAsGetMiddleware = SymbolUtils.createValueSymbolBuilder("AddAsGetRequestMiddleware", + AwsGoDependency.AWS_QUERY_PROTOCOL).build(); + + writer.writeDocs("convert request to a GET request"); + writer.write("err = $T(stack)", queryAsGetMiddleware); writer.write("if err != nil { return err }"); - }); + } + + // s3 service needs expires + if (isS3ServiceShape(model, serviceShape)) { + Symbol expiresAsHeaderMiddleware = SymbolUtils.createValueSymbolBuilder( + "AddExpiresOnPresignedURL", + AwsCustomGoDependency.S3_CUSTOMIZATION).build(); + writer.openBlock("if c.expires != 0 {", "}", () -> { + writer.writeDocs("add middleware to set expiration for s3 presigned url"); + writer.write("err = stack.Build.Add(&$T{ Expires: c.expires, }, middleware.After)", + expiresAsHeaderMiddleware); + writer.write("if err != nil { return err }"); + }); + } writer.write("return nil"); - }); + }).insertTrailingNewline(); } - public Symbol getPresignClientSymbol() { - return presignClientSymbol; - } - public Symbol getNewPresignClientSymbol() { - return newPresignClientSymbol; - } + /** + * Writes the Presign client's type and methods. + * + * @param writer writer to write to + */ + private void writePresignClientType( + GoWriter writer, + Model model, + SymbolProvider symbolProvider, + ServiceShape serviceShape + ) { + writer.addUseImports(SmithyGoDependency.CONTEXT); + writer.addUseImports(AwsGoDependency.AWS_SIGNER_V4); - private static Symbol buildNewPresignClientSymbol(Symbol operation, boolean exported) { - String name = String.format("New%sHTTPPresignURLClient", operation.getName()); - return buildSymbol(name, exported); - } - private static Symbol buildPresignClientSymbol(Symbol operation, boolean exported) { - String name = String.format("%sHTTPPresignURLClient", operation.getName()); - return buildSymbol(name, exported); - } + Symbol presignerInterfaceSymbol = SymbolUtils.createPointableSymbolBuilder( + "HTTPPresigner", AwsGoDependency.AWS_SIGNER_V4 + ).build(); + + writer.writeDocs(String.format("%s represents the presign url client", PRESIGN_CLIENT)); + writer.openBlock("type $T struct {", "}", presignClientSymbol, () -> { + writer.write("client *Client"); + writer.write("presigner v4.HTTPPresigner"); + + if (isS3ServiceShape(model, serviceShape)) { + writer.addUseImports(SmithyGoDependency.TIME); + writer.write("expires time.Duration"); + } - private static Symbol buildAPIClientSymbol(Symbol operation, boolean exported) { - String name = String.format("%sAPIClient", operation.getName()); - return buildSymbol(name, exported); + }); + + // generate constructors + + // generate NewPresignClient + writer.writeDocs( + String.format("%s generates a presign client using provided Client options and presign options", + NEW_CLIENT) + ); + writer.openBlock("func $L(options Options, optFns ...func($P)) $P {", "}", + NEW_CLIENT, presignOptionsSymbol, presignClientSymbol, () -> { + processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); + writer.insertTrailingNewline(); + + writer.write("client := New(options, presignOptions.ClientOptions...)").insertTrailingNewline(); + writer.insertTrailingNewline(); + + returnPresignClientConstructor(writer, model, serviceShape); + }).insertTrailingNewline(); + + // generate NewPresignClientWrapper + writer.writeDocs( + String.format("%s generates a presign client using provided API Client and presign options", + NEW_CLIENT_FROM_SERVICE) + ); + writer.openBlock("func $L(c *Client, optFns ...func($P)) $P {", "}", + NEW_CLIENT_FROM_SERVICE, presignOptionsSymbol, presignClientSymbol, () -> { + processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); + writer.insertTrailingNewline(); + + writer.write("client := copyAPIClient(c, presignOptions.ClientOptions...)"); + writer.insertTrailingNewline(); + + returnPresignClientConstructor(writer, model, serviceShape); + }).insertTrailingNewline(); + + // generate NewPresignClientFromConfig + writer.writeDocs( + String.format("%s generates a presign client using provided AWS config and presign options", + NEW_CLIENT_FROM_CONFIG) + ); + writer.openBlock("func $L(cfg aws.Config, optFns ...func($P)) $P {", "}", + NEW_CLIENT_FROM_CONFIG, presignOptionsSymbol, presignClientSymbol, () -> { + processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); + writer.insertTrailingNewline(); + + writer.write("client := NewFromConfig(cfg, presignOptions.ClientOptions...)"); + writer.insertTrailingNewline(); + + returnPresignClientConstructor(writer, model, serviceShape); + }).insertTrailingNewline(); + + writer.insertTrailingNewline(); } - private static Symbol buildSymbol(String name, boolean exported) { - if (!exported) { - name = Character.toLowerCase(name.charAt(0)) + name.substring(1); - } - return SymbolUtils.createValueSymbolBuilder(name). - build(); + /** + * Writes the Presign client's helper methods. + * + * @param writer writer to write to + */ + private void writePresignClientHelpers( + GoWriter writer, + Model model, + SymbolProvider symbolProvider, + ServiceShape serviceShape + ) { + // generate copy API client + final String COPY_API_CLIENT = "copyAPIClient"; + writer.openBlock("func $L(c *Client, optFns ...func(*Options)) *Client {", "}", + COPY_API_CLIENT, () -> { + writer.write("return New(c.options, optFns...)"); + writer.insertTrailingNewline(); + }).insertTrailingNewline(); + writer.insertTrailingNewline(); } /** - * Builder for the HTTP Presign URL client client generator. + * Writes the Presign client's type and methods. + * + * @param writer writer to write to */ - public static class Builder implements SmithyBuilder { - private Model model; - private SymbolProvider symbolProvider; - private OperationShape operation; - private boolean exported; - private List convertToPresignMiddlewareHelpers = new ArrayList<>(); - - /** - * Sets the model for the builder - * - * @param model API model - * @return builder - */ - public Builder model(Model model) { - this.model = model; - return this; - } + public void writePresignOptionType( + GoWriter writer, + Model model, + SymbolProvider symbolProvider, + ServiceShape serviceShape + ) { + writer.addUseImports(SmithyGoDependency.CONTEXT); + writer.addUseImports(AwsGoDependency.AWS_SIGNER_V4); - /** - * Sets the symbol provider for the builder - * - * @param symbolProvider the symbol provider - * @return buidler - */ - public Builder symbolProvider(SymbolProvider symbolProvider) { - this.symbolProvider = symbolProvider; - return this; - } + Symbol presignOptionSymbol = buildSymbol(PRESIGN_OPTIONS, true); - /** - * Sets the operation for the builder - * - * @param operation api operation - * @return builder - */ - public Builder operation(OperationShape operation) { - this.operation = operation; - return this; - } + // generate presign options + writer.writeDocs(String.format("%s represents the presign client options", PRESIGN_OPTIONS)); + writer.openBlock("type $T struct {", "}", presignOptionSymbol, () -> { + writer.writeDocs( + "ClientOptions are list of functional options to mutate client options used by presign client"); + writer.write("ClientOptions []func(*Options)"); + writer.insertTrailingNewline(); - /** - * Sets that the generated client type should be exported, defaults to false. - * - * @return builder - */ - public Builder exported() { - return this.exported(true); - } + writer.writeDocs("Presigner is the presigner used by the presign url client"); + writer.write("Presigner $T", presignerInterfaceSymbol); - /** - * Sets if the generate client type should be exported or not. - * - * @param exported if exported - * @return builder - */ - public Builder exported(boolean exported) { - this.exported = exported; - return this; - } + // s3 service has an additional Expires options + if (isS3ServiceShape(model, serviceShape)) { + writer.writeDocs("Expires sets the expiration duration for the generated presign url"); + writer.write("Expires time.Duration"); + } + }); - /** - * Sets additional middleware mutator that will be generated into the client's convert to presign URL operation. - * Used by the client to convert a API operation to a presign URL. - * - * @param middlewareHelpers list of middleware helpers to set - * @return builder - */ - public Builder convertToPresignMiddlewareHelpers(List middlewareHelpers) { - this.convertToPresignMiddlewareHelpers.clear(); - this.convertToPresignMiddlewareHelpers.addAll(middlewareHelpers); - return this; - } + // generate WithPresignClientFromClientOptions Helper + Symbol presignOptionsFromClientOptionsInternal = buildSymbol(PRESIGN_OPTIONS_FROM_CLIENT_OPTIONS, false); + writer.writeDocs( + String.format("%s is a helper utility to retrieve a function that takes PresignOption as input", + PRESIGN_OPTIONS_FROM_CLIENT_OPTIONS) + ); + writer.openBlock("func $L(optFns ...func(*Options)) func($P) {", "}", + PRESIGN_OPTIONS_FROM_CLIENT_OPTIONS, presignOptionSymbol, () -> { + writer.write("return $L(optFns).options", presignOptionsFromClientOptionsInternal.getName()); + }); - /** - * Adds a single middleware mutator that will be generated into the client's convert to presign URL operation. - * Used by the client to convert API operation to a presigned URL. - * - * @param middlewareHelper the middleware helper to add - * @return builder - */ - public Builder addConvertToPresignMiddlewareHelpers(Symbol middlewareHelper) { - this.convertToPresignMiddlewareHelpers.add(middlewareHelper); - return this; - } + writer.insertTrailingNewline(); + + writer.write("type $L []func(*Options)", presignOptionsFromClientOptionsInternal.getName()); + writer.openBlock("func (w $L) options (o $P) {", "}", + presignOptionsFromClientOptionsInternal.getName(), presignOptionSymbol, () -> { + writer.write("o.ClientOptions = append(o.ClientOptions, w...)"); + }).insertTrailingNewline(); + + + // s3 specific helpers + if (isS3ServiceShape(model, serviceShape)) { + // generate WithPresignExpires Helper + Symbol presignOptionsFromExpiresInternal = buildSymbol(PRESIGN_OPTIONS_FROM_EXPIRES, false); + writer.writeDocs(String.format( + "%s is a helper utility to append Expires value on presign options optional function", + PRESIGN_OPTIONS_FROM_EXPIRES)); + writer.openBlock("func $L(dur time.Duration) func($P) {", "}", + PRESIGN_OPTIONS_FROM_EXPIRES, presignOptionSymbol, () -> { + writer.write("return $L(dur).options", presignOptionsFromExpiresInternal.getName()); + }); - // TODO presign with expire can be supported with a builder param that enables expires param behavior. + writer.insertTrailingNewline(); - public AwsHttpPresignURLClientGenerator build() { - return new AwsHttpPresignURLClientGenerator(this); + writer.write("type $L time.Duration", presignOptionsFromExpiresInternal.getName()); + writer.openBlock("func (w $L) options (o $P) {", "}", + presignOptionsFromExpiresInternal.getName(), presignOptionSymbol, () -> { + writer.write("o.Expires = time.Duration(w)"); + }).insertTrailingNewline(); } } + + private final boolean isS3ServiceShape(Model model, ServiceShape service) { + String serviceId = service.expectTrait(ServiceTrait.class).getSdkId(); + return serviceId.equalsIgnoreCase("S3"); + } } + +// TODO: generate tests for presigned urls From d8198fa784999515a315bbcdcf33de989080a24f Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 11 Nov 2020 01:44:30 -0800 Subject: [PATCH 02/24] register runtime client plugin --- ...ftware.amazon.smithy.go.codegen.integration.GoIntegration | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index a45b33d3479..5a88e0f1191 100644 --- a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -10,6 +10,9 @@ software.amazon.smithy.aws.go.codegen.AwsSdkServiceId software.amazon.smithy.aws.go.codegen.AwsRetryMiddlewareHelper software.amazon.smithy.aws.go.codegen.AWSRequestIDRetriever software.amazon.smithy.aws.go.codegen.AWSResponseErrorWrapper +software.amazon.smithy.aws.go.codegen.AwsHttpPresignURLClientGenerator +software.amazon.smithy.aws.go.codegen.FilterStreamingOperations +software.amazon.smithy.aws.go.codegen.RequestResponseLogging software.amazon.smithy.aws.go.codegen.customization.DynamoDBValidateResponseChecksum software.amazon.smithy.aws.go.codegen.customization.S3UpdateEndpoint software.amazon.smithy.aws.go.codegen.customization.APIGatewayAcceptHeader @@ -26,6 +29,4 @@ software.amazon.smithy.aws.go.codegen.customization.S3ErrorWith200Status software.amazon.smithy.aws.go.codegen.customization.Route53Customizations software.amazon.smithy.aws.go.codegen.customization.S3HeadObjectCustomizations software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill -software.amazon.smithy.aws.go.codegen.FilterStreamingOperations -software.amazon.smithy.aws.go.codegen.RequestResponseLogging software.amazon.smithy.aws.go.codegen.customization.S3ControlEndpointResolver From fdb62b4083817d050e17e94b3bb3bb06cd74a3f5 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 11 Nov 2020 01:45:25 -0800 Subject: [PATCH 03/24] modify java codegen for existing rds autofill customization to use new presign clients --- .../customization/PresignURLAutoFill.java | 250 +++++++++--------- 1 file changed, 120 insertions(+), 130 deletions(-) diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java index ef93b8c3de5..ad4fd7ede1e 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java @@ -24,8 +24,6 @@ import java.util.TreeMap; import java.util.logging.Logger; import software.amazon.smithy.aws.go.codegen.AwsGoDependency; -import software.amazon.smithy.aws.go.codegen.AwsHttpPresignURLClientGenerator; -import software.amazon.smithy.codegen.core.CodegenException; import software.amazon.smithy.codegen.core.Symbol; import software.amazon.smithy.codegen.core.SymbolProvider; import software.amazon.smithy.go.codegen.CodegenUtils; @@ -56,9 +54,6 @@ public class PresignURLAutoFill implements GoIntegration { private static final Logger LOGGER = Logger.getLogger(PresignURLAutoFill.class.getName()); - - private final List runtimeClientPlugins = new ArrayList<>(); - private static final Map> SERVICE_TO_OPERATION_MAP = MapUtils.of( ShapeId.from("com.amazonaws.rds#AmazonRDSv19"), SetUtils.of( ShapeId.from("com.amazonaws.rds#CopyDBSnapshot"), @@ -71,6 +66,121 @@ public class PresignURLAutoFill implements GoIntegration { // TODO other services ); + private final List runtimeClientPlugins = new ArrayList<>(); + + private static void writeMemberSetter( + GoWriter writer, + SymbolProvider symbolprovider, + OperationShape operation, + StructureShape input, + MemberShape member + ) { + Symbol operationSymbol = symbolprovider.toSymbol(operation); + Symbol inputSymbol = symbolprovider.toSymbol(input); + String memberName = symbolprovider.toMemberName(member); + + writer.openBlock("func $L(params interface{}, value string) error {", "}", + setterFuncName(operationSymbol.getName(), memberName), + () -> { + writer.addUseImports(SmithyGoDependency.FMT); + writer.write("input, ok := params.($P)", inputSymbol); + writer.openBlock("if !ok {", "}", () -> { + writer.write("return fmt.Errorf(\"expect $P type, got %T\", params)", inputSymbol); + }); + writer.write("input.$L = &value", memberName); + writer.write("return nil"); + }); + } + + private static void writeMemberGetter( + GoWriter writer, + SymbolProvider symbolprovider, + OperationShape operation, + StructureShape input, + MemberShape member + ) { + Symbol operationSymbol = symbolprovider.toSymbol(operation); + Symbol inputSymbol = symbolprovider.toSymbol(input); + String memberName = symbolprovider.toMemberName(member); + + writer.openBlock("func $L(params interface{}) (string, bool, error) {", "}", + getterFuncName(operationSymbol.getName(), memberName), + () -> { + writer.addUseImports(SmithyGoDependency.FMT); + writer.write("input, ok := params.($P)", inputSymbol); + writer.openBlock("if !ok {", "}", () -> { + writer.write("return ``, false, fmt.Errorf(\"expect $P type, got %T\", params)", inputSymbol); + }); + writer.openBlock("if input.$L == nil || len(*input.$L) == 0 {", "}", memberName, memberName, + () -> writer.write("return ``, false, nil") + ); + writer.write("return *input.$L, true, nil", memberName); + }); + } + + private static void writePresignOperationAccessor( + GoWriter writer, + SymbolProvider symbolprovider, + OperationShape operation, + StructureShape input + ) { + Symbol operationSymbol = symbolprovider.toSymbol(operation); + Symbol inputSymbol = symbolprovider.toSymbol(input); + + Symbol removeMiddleware = SymbolUtils.createValueSymbolBuilder("RemoveMiddleware", + AwsCustomGoDependency.PRESIGNEDURL_CUSTOMIZATION).build(); + + writer.openBlock( + "func $L(ctx context.Context, client interface{}, region string, params interface{}) " + + "(req *v4.PresignedHTTPRequest, err error) {", "}", + presignFuncName(operationSymbol.getName(), false), + () -> { + writer.addUseImports(SmithyGoDependency.FMT); + // check input + writer.write("input, ok := params.($P)", inputSymbol); + writer.openBlock("if !ok {", "}", () -> { + writer.write("return req, fmt.Errorf(\"expect $P type, got %T\", params)", inputSymbol); + }); + + // check client + writer.write("c, ok := client.(*PresignClient)"); + writer.openBlock("if !ok {", "}", () -> { + writer.write("return req, fmt.Errorf(\"expect *PresignClient type, got %T\", client)"); + }); + + // generate client options + writer.openBlock("optFn := func(o *Options) {", "}", () -> { + writer.write("o.Region = region"); + writer.write("o.APIOptions = append(o.APIOptions, $T)", removeMiddleware); + }); + + // getPresignAPIOptions + writer.write("presignOptFn := WithPresignClientFromClientOptions(optFn)"); + + // call the exported function + writer.write("return c.$L(ctx, input, presignOptFn)", presignFuncName(operationSymbol.getName(), true)); + }); + } + + private static String addPresignMiddlewareFuncName(String operationName) { + return String.format("add%sPresignURLMiddleware", operationName); + } + + private static String getterFuncName(String operationName, String memberName) { + return String.format("get%s%s", operationName, memberName); + } + + private static String setterFuncName(String operationName, String memberName) { + return String.format("set%s%s", operationName, memberName); + } + + private static String copyInputFuncName(String inputName) { + return String.format("copy%sForPresign", inputName); + } + + private static String presignFuncName(String operationName, boolean exported) { + return exported ? String.format("Presign%s", operationName) : String.format("presign%s", operationName); + } /** * Updates the API model to add additional members to the operation input shape that are needed for presign url @@ -161,6 +271,7 @@ public void writeAdditionalFiles( // Members used by the customization need abstract getter and setters writeMemberAccessor(writer, symbolProvider, operation, input); + writePresignOperationAccessor(writer, symbolProvider, operation, input); // Generate the presign client writePresignClientCustomization(writer, settings, model, symbolProvider, operation, input); @@ -250,56 +361,6 @@ private void writeMemberAccessor( } } - private static void writeMemberSetter( - GoWriter writer, - SymbolProvider symbolprovider, - OperationShape operation, - StructureShape input, - MemberShape member - ) { - Symbol operationSymbol = symbolprovider.toSymbol(operation); - Symbol inputSymbol = symbolprovider.toSymbol(input); - String memberName = symbolprovider.toMemberName(member); - - writer.openBlock("func $L(params interface{}, value string) error {", "}", - setterFuncName(operationSymbol.getName(), memberName), - () -> { - writer.addUseImports(SmithyGoDependency.FMT); - writer.write("input, ok := params.($P)", inputSymbol); - writer.openBlock("if !ok {", "}", () -> { - writer.write("return fmt.Errorf(\"expect $P type, got %T\", params)", inputSymbol); - }); - writer.write("input.$L = &value", memberName); - writer.write("return nil"); - }); - } - - private static void writeMemberGetter( - GoWriter writer, - SymbolProvider symbolprovider, - OperationShape operation, - StructureShape input, - MemberShape member - ) { - Symbol operationSymbol = symbolprovider.toSymbol(operation); - Symbol inputSymbol = symbolprovider.toSymbol(input); - String memberName = symbolprovider.toMemberName(member); - - writer.openBlock("func $L(params interface{}) (string, bool, error) {", "}", - getterFuncName(operationSymbol.getName(), memberName), - () -> { - writer.addUseImports(SmithyGoDependency.FMT); - writer.write("input, ok := params.($P)", inputSymbol); - writer.openBlock("if !ok {", "}", () -> { - writer.write("return ``, false, fmt.Errorf(\"expect $P type, got %T\", params)", inputSymbol); - }); - writer.openBlock("if input.$L == nil || len(*input.$L) == 0 {", "}", memberName, memberName, - () -> writer.write("return ``, false, nil") - ); - writer.write("return *input.$L, true, nil", memberName); - }); - } - private void writePresignClientCustomization( GoWriter writer, GoSettings settings, @@ -308,29 +369,6 @@ private void writePresignClientCustomization( OperationShape operation, StructureShape input ) { - AwsHttpPresignURLClientGenerator.Builder clientGenBuilder = new AwsHttpPresignURLClientGenerator.Builder() - .model(model) - .symbolProvider(symbolProvider) - .operation(operation); - - switch (settings.getProtocol().toString()) { - case "aws.protocols#awsQuery": - case "aws.protocols#ec2Query": - Symbol queryAsGetMiddleware = SymbolUtils.createValueSymbolBuilder("AddAsGetRequestMiddleware", - AwsGoDependency.AWS_QUERY_PROTOCOL) - .build(); - clientGenBuilder.addConvertToPresignMiddlewareHelpers(queryAsGetMiddleware); - break; - - default: - LOGGER.warning("presign url customization does not know how to handle service " - + settings.getService() + " using protocol "+ settings.getProtocol()); - } - - AwsHttpPresignURLClientGenerator gen = clientGenBuilder.build(); - - gen.writePresignClientType(writer); - Symbol smithyStack = SymbolUtils.createPointableSymbolBuilder("Stack", SmithyGoDependency.SMITHY_MIDDLEWARE).build(); Symbol operationSymbol = symbolProvider.toSymbol(operation); @@ -347,8 +385,7 @@ private void writePresignClientCustomization( AwsCustomGoDependency.PRESIGNEDURL_CUSTOMIZATION).build(); Symbol addMiddleware = SymbolUtils.createValueSymbolBuilder("AddMiddleware", AwsCustomGoDependency.PRESIGNEDURL_CUSTOMIZATION).build(); - Symbol removeMiddleware = SymbolUtils.createValueSymbolBuilder("RemoveMiddleware", - AwsCustomGoDependency.PRESIGNEDURL_CUSTOMIZATION).build(); + // generate middleware mutator to wire up presign client with accessors and custom middleware. writer.openBlock("func $L(stack $P, options Options) error {", "}", @@ -366,40 +403,13 @@ private void writePresignClientCustomization( setterFuncName(operationSymbol.getName(), dstRegionMember)); writer.write("SetPresignedURL: $L,", setterFuncName(operationSymbol.getName(), presignURLMember)); + writer.write("PresignOperation: $L,", + presignFuncName(operationSymbol.getName(), false)); }); // Replace with type wrapping presigner for generic signature - writer.write("Presigner: &$L{client: $T(options)},", - opPresignClientWrapperName(operationSymbol.getName()), - gen.getNewPresignClientSymbol()); + writer.write("PresignClient: NewPresignClient(options),"); }); }); - - // Generate generic presign wrapper type for passing region in with op call. - writer.openBlock("type $L struct {", "}", - opPresignClientWrapperName(operationSymbol.getName()), - () -> { - writer.write("client *$T", gen.getPresignClientSymbol()); - }); - - writer.addUseImports(SmithyGoDependency.NET_HTTP); - writer.openBlock( - // TODO consider creating type for presign parameters for future compatibility. - "func (c *$L) PresignURL(ctx context.Context, region string, params interface{}) " - + "(string, http.Header, error) {", "}", - opPresignClientWrapperName(operationSymbol.getName()), - () -> { - writer.write("input, ok := params.($P)", inputSymbol); - writer.openBlock("if !ok {", "}", () -> { - writer.write("return ``, nil, fmt.Errorf(\"expect $P type, got %T\", params)", inputSymbol); - }); - - // TODO could be replaced with a `WithRegion` client option helper. - writer.openBlock("optFn := func(o *Options) {", "}", () -> { - writer.write("o.Region = region"); - writer.write("o.APIOptions = append(o.APIOptions, $T)", removeMiddleware); - }); - writer.write("return c.client.Presign$L(ctx, input, optFn)", operationSymbol.getName()); - }); } private void writePresignClientCustomizationTest( @@ -439,24 +449,4 @@ private void writePresignClientCustomizationTest( writer.write(template); } - - private static String opPresignClientWrapperName(String operationName) { - return String.format("presignAutoFill%sClient", operationName); - } - - private static String addPresignMiddlewareFuncName(String operationName) { - return String.format("add%sPresignURLMiddleware", operationName); - } - - private static String getterFuncName(String operationName, String memberName) { - return String.format("get%s%s", operationName, memberName); - } - - private static String setterFuncName(String operationName, String memberName) { - return String.format("set%s%s", operationName, memberName); - } - - private static String copyInputFuncName(String inputName) { - return String.format("copy%sForPresign", inputName); - } } From 552f0c50ac93ceaf7c01a1953652c74063cef6e5 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 11 Nov 2020 01:47:56 -0800 Subject: [PATCH 04/24] internal/presignedURL pkg changes to support new design. This is internal package, so changes are not considered as breaking --- service/internal/presigned-url/middleware.go | 36 +++++++++++-------- .../internal/presigned-url/middleware_test.go | 16 ++++++--- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/service/internal/presigned-url/middleware.go b/service/internal/presigned-url/middleware.go index 8caf0219600..9e732f45e88 100644 --- a/service/internal/presigned-url/middleware.go +++ b/service/internal/presigned-url/middleware.go @@ -3,35 +3,43 @@ package presignedurl import ( "context" "fmt" - "net/http" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/awslabs/smithy-go/middleware" ) // ParameterAccessor provides an collection of accessor to for retrieving and // setting the values needed to PresignedURL generation type ParameterAccessor struct { + // GetPresignedURL accessor points to a function that retrieves a presigned url if present GetPresignedURL func(interface{}) (string, bool, error) + + // GetSourceRegion accessor points to a function that retrieves source region for presigned url GetSourceRegion func(interface{}) (string, bool, error) - CopyInput func(interface{}) (interface{}, error) + // CopyInput accessor points to a function that takes in an input, and returns a copy. + CopyInput func(interface{}) (interface{}, error) + + // SetDestinationRegion accessor points to a function that sets destination region on api input struct SetDestinationRegion func(interface{}, string) error - SetPresignedURL func(interface{}, string) error -} -// URLPresigner provides the interface to presign the input parameters in to a -// presigned URL. -type URLPresigner interface { - PresignURL(ctx context.Context, srcRegion string, params interface{}) ( - presignedURL string, signedHeader http.Header, err error, - ) + // SetPresignedURL accessor points to a function that sets presigned url on api input struct + SetPresignedURL func(interface{}, string) error + + // PresignOperation is the presign function accessor + PresignOperation func(ctx context.Context, client interface{}, + srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error) } // Options provides the set of options needed by the presigned URL middleware. type Options struct { - Accessor ParameterAccessor - Presigner URLPresigner + // Accessor are the parameter accessors used by this middleware + Accessor ParameterAccessor + + // PresignClient is the presigner client used to presign an operation request. + PresignClient interface{} } // AddMiddleware adds the Presign URL middleware to the middleware stack. @@ -85,13 +93,13 @@ func (m *presign) HandleInitialize( return out, metadata, fmt.Errorf("presign middleware failed, %w", err) } - presignedURL, _, err := m.options.Presigner.PresignURL(ctx, srcRegion, paramCpy) + presignedReq, err := m.options.Accessor.PresignOperation(ctx, m.options.PresignClient, srcRegion, paramCpy) if err != nil { return out, metadata, fmt.Errorf("unable to create presigned URL, %w", err) } // Update the original input with the presigned URL value. - if err = m.options.Accessor.SetPresignedURL(input.Parameters, presignedURL); err != nil { + if err = m.options.Accessor.SetPresignedURL(input.Parameters, presignedReq.URL); err != nil { return out, metadata, fmt.Errorf("presign middleware failed, %w", err) } diff --git a/service/internal/presigned-url/middleware_test.go b/service/internal/presigned-url/middleware_test.go index a81a5b1d83b..b3ce8e669a0 100644 --- a/service/internal/presigned-url/middleware_test.go +++ b/service/internal/presigned-url/middleware_test.go @@ -7,6 +7,8 @@ import ( "testing" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" "github.com/google/go-cmp/cmp" @@ -123,8 +125,9 @@ func getURLPresignMiddlewareOptions() Options { c.(*mockURLPresignInput).PresignedURL = v return nil }, + PresignOperation: presignURL, }, - Presigner: mockURLPresigner{}, + PresignClient: mockURLPresigner{}, } } @@ -136,11 +139,14 @@ type mockURLPresignInput struct { type mockURLPresigner struct{} -func (mockURLPresigner) PresignURL(ctx context.Context, srcRegion string, params interface{}) ( - presignedURL string, signedHeader http.Header, err error, +func presignURL(ctx context.Context, client interface{}, srcRegion string, params interface{}) ( + req *v4.PresignedHTTPRequest, err error, ) { in := params.(*mockURLPresignInput) - return "https://example." + srcRegion + ".amazonaws.com/?DestinationRegion=" + in.DestinationRegion, - http.Header{}, nil + return &v4.PresignedHTTPRequest{ + URL: "https://example." + srcRegion + ".amazonaws.com/?DestinationRegion=" + in.DestinationRegion, + Method: "POST", + SignedHeader: http.Header{}, + }, nil } From ab85f51a91c9b36b998ae38e5c5ea88c471b1614 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 11 Nov 2020 01:48:43 -0800 Subject: [PATCH 05/24] add a build middleware to handle s3 use of expires option for presigned urls --- .../customizations/presigned_expires.go | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 service/s3/internal/customizations/presigned_expires.go diff --git a/service/s3/internal/customizations/presigned_expires.go b/service/s3/internal/customizations/presigned_expires.go new file mode 100644 index 00000000000..2890e7df5d6 --- /dev/null +++ b/service/s3/internal/customizations/presigned_expires.go @@ -0,0 +1,45 @@ +package customizations + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/awslabs/smithy-go/middleware" + smithyhttp "github.com/awslabs/smithy-go/transport/http" +) + +// AddExpiresOnPresignedURL represents a build middleware used to assign +// expiration on a presigned URL +type AddExpiresOnPresignedURL struct { + // Expires is time.Duration within which presigned url should be expired + Expires time.Duration +} + +// ID representing the middleware +func (*AddExpiresOnPresignedURL) ID() string { + return "S3:AddExpiresOnPresignedURL" +} + +// HandleBuild handles the build step middleware behavior +func (m *AddExpiresOnPresignedURL) HandleBuild(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + // if expiration is unset skip this middleware + if m.Expires == 0 { + return next.HandleBuild(ctx, in) + } + + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown transport type %T", req) + } + + // set S3 X-AMZ-Expires header + query := req.URL.Query() + query.Set("X-Amz-Expires", strconv.FormatInt(int64(m.Expires/time.Second), 10)) + req.URL.RawQuery = query.Encode() + + return next.HandleBuild(ctx, in) +} From 0430dfbc76c20ddb0f9fce8587472431cb8baaa6 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 11 Nov 2020 02:01:06 -0800 Subject: [PATCH 06/24] fix formatting --- .../aws/go/codegen/customization/PresignURLAutoFill.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java index ad4fd7ede1e..b1a109b69f7 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java @@ -150,7 +150,7 @@ private static void writePresignOperationAccessor( // generate client options writer.openBlock("optFn := func(o *Options) {", "}", () -> { - writer.write("o.Region = region"); + writer.write("o.Region = region"); writer.write("o.APIOptions = append(o.APIOptions, $T)", removeMiddleware); }); @@ -158,7 +158,8 @@ private static void writePresignOperationAccessor( writer.write("presignOptFn := WithPresignClientFromClientOptions(optFn)"); // call the exported function - writer.write("return c.$L(ctx, input, presignOptFn)", presignFuncName(operationSymbol.getName(), true)); + writer.write("return c.$L(ctx, input, presignOptFn)", + presignFuncName(operationSymbol.getName(), true)); }); } From c7da36562d76e4de91765efd5b50ddc9f7f53ae5 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 11 Nov 2020 02:02:16 -0800 Subject: [PATCH 07/24] generated s3, rds, ec2 client for verification --- service/ec2/api_client.go | 103 +++++++++++++++ service/ec2/api_op_CopySnapshot.go | 103 ++++++--------- service/rds/api_client.go | 103 +++++++++++++++ service/rds/api_op_CopyDBClusterSnapshot.go | 103 ++++++--------- service/rds/api_op_CopyDBSnapshot.go | 103 ++++++--------- service/rds/api_op_CreateDBCluster.go | 103 ++++++--------- .../rds/api_op_CreateDBInstanceReadReplica.go | 103 ++++++--------- service/s3/api_client.go | 122 ++++++++++++++++++ service/s3/api_op_PutObject.go | 28 ++++ service/s3/go.mod | 3 + service/s3/go.sum | 2 + service/s3/internal/configtesting/go.mod | 2 + service/s3/internal/configtesting/go.sum | 2 + 13 files changed, 570 insertions(+), 310 deletions(-) diff --git a/service/ec2/api_client.go b/service/ec2/api_client.go index 5b0ed68bc6a..36ccfb94e7b 100644 --- a/service/ec2/api_client.go +++ b/service/ec2/api_client.go @@ -7,6 +7,7 @@ import ( cryptorand "crypto/rand" "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/aws-sdk-go-v2/aws/protocol/query" "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" @@ -246,3 +247,105 @@ func addRequestResponseLogging(stack *middleware.Stack, o Options) error { LogResponseWithBody: o.ClientLogMode.IsResponseWithBody(), }, middleware.After) } + +// PresignOptions represents the presign client options +type PresignOptions struct { + // ClientOptions are list of functional options to mutate client options used by + // presign client + ClientOptions []func(*Options) + // Presigner is the presigner used by the presign url client + Presigner v4.HTTPPresigner +} + +// WithPresignClientFromClientOptions is a helper utility to retrieve a function +// that takes PresignOption as input +func WithPresignClientFromClientOptions(optFns ...func(*Options)) func(*PresignOptions) { + return withPresignClientFromClientOptions(optFns).options +} + +type withPresignClientFromClientOptions []func(*Options) + +func (w withPresignClientFromClientOptions) options(o *PresignOptions) { + o.ClientOptions = append(o.ClientOptions, w...) +} + +// PresignClient represents the presign url client +type PresignClient struct { + client *Client + presigner v4.HTTPPresigner +} + +// NewPresignClient generates a presign client using provided Client options and +// presign options +func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := New(options, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + } +} + +// NewPresignClientWrapper generates a presign client using provided API Client and +// presign options +func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := copyAPIClient(c, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + } +} + +// NewPresignClientFromConfig generates a presign client using provided AWS config +// and presign options +func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := NewFromConfig(cfg, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + } +} +func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { + return New(c.options, optFns...) +} +func (c *PresignClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { + stack.Finalize.Clear() + stack.Deserialize.Clear() + stack.Build.Remove((*awsmiddleware.ClientRequestID)(nil).ID()) + err = stack.Finalize.Add(v4.NewPresignHTTPRequestMiddleware(options.Credentials, c.presigner), middleware.After) + if err != nil { + return err + } + // convert request to a GET request + err = query.AddAsGetRequestMiddleware(stack) + if err != nil { + return err + } + return nil +} diff --git a/service/ec2/api_op_CopySnapshot.go b/service/ec2/api_op_CopySnapshot.go index 9426ea2a80a..2e38ecebd9b 100644 --- a/service/ec2/api_op_CopySnapshot.go +++ b/service/ec2/api_op_CopySnapshot.go @@ -6,13 +6,11 @@ import ( "context" "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" - "github.com/aws/aws-sdk-go-v2/aws/protocol/query" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/aws-sdk-go-v2/service/ec2/types" presignedurlcust "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url" "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" - "net/http" ) // Copies a point-in-time snapshot of an EBS volume and stores it in Amazon S3. You @@ -240,52 +238,21 @@ func setCopySnapshotdestinationRegion(params interface{}, value string) error { input.destinationRegion = &value return nil } - -type copySnapshotHTTPPresignURLClient struct { - client *Client - presigner *v4.Signer -} - -func newCopySnapshotHTTPPresignURLClient(options Options, optFns ...func(*Options)) *copySnapshotHTTPPresignURLClient { - return ©SnapshotHTTPPresignURLClient{ - client: New(options, optFns...), - presigner: v4.NewSigner(), +func presignCopySnapshot(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { + input, ok := params.(*CopySnapshotInput) + if !ok { + return req, fmt.Errorf("expect *CopySnapshotInput type, got %T", params) } -} -func (c *copySnapshotHTTPPresignURLClient) PresignCopySnapshot(ctx context.Context, params *CopySnapshotInput, optFns ...func(*Options)) (string, http.Header, error) { - if params == nil { - params = &CopySnapshotInput{} + c, ok := client.(*PresignClient) + if !ok { + return req, fmt.Errorf("expect *PresignClient type, got %T", client) } - - optFns = append(optFns, func(o *Options) { - o.HTTPClient = &smithyhttp.NopClient{} - }) - - ctx = presignedurlcust.WithIsPresigning(ctx) - result, _, err := c.client.invokeOperation(ctx, "CopySnapshot", params, optFns, - addOperationCopySnapshotMiddlewares, - c.convertToPresignMiddleware, - ) - if err != nil { - return ``, nil, err + optFn := func(o *Options) { + o.Region = region + o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } - - out := result.(*v4.PresignedHTTPRequest) - return out.URL, out.SignedHeader, nil -} -func (c *copySnapshotHTTPPresignURLClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { - stack.Finalize.Clear() - stack.Deserialize.Clear() - stack.Build.Remove((*awsmiddleware.ClientRequestID)(nil).ID()) - err = stack.Finalize.Add(v4.NewPresignHTTPRequestMiddleware(options.Credentials, c.presigner), middleware.After) - if err != nil { - return err - } - err = query.AddAsGetRequestMiddleware(stack) - if err != nil { - return err - } - return nil + presignOptFn := WithPresignClientFromClientOptions(optFn) + return c.PresignCopySnapshot(ctx, input, presignOptFn) } func addCopySnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ @@ -295,27 +262,12 @@ func addCopySnapshotPresignURLMiddleware(stack *middleware.Stack, options Option CopyInput: copyCopySnapshotInputForPresign, SetDestinationRegion: setCopySnapshotdestinationRegion, SetPresignedURL: setCopySnapshotPresignedUrl, + PresignOperation: presignCopySnapshot, }, - Presigner: &presignAutoFillCopySnapshotClient{client: newCopySnapshotHTTPPresignURLClient(options)}, + PresignClient: NewPresignClient(options), }) } -type presignAutoFillCopySnapshotClient struct { - client *copySnapshotHTTPPresignURLClient -} - -func (c *presignAutoFillCopySnapshotClient) PresignURL(ctx context.Context, region string, params interface{}) (string, http.Header, error) { - input, ok := params.(*CopySnapshotInput) - if !ok { - return ``, nil, fmt.Errorf("expect *CopySnapshotInput type, got %T", params) - } - optFn := func(o *Options) { - o.Region = region - o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) - } - return c.client.PresignCopySnapshot(ctx, input, optFn) -} - func newServiceMetadataMiddleware_opCopySnapshot(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, @@ -324,3 +276,30 @@ func newServiceMetadataMiddleware_opCopySnapshot(region string) *awsmiddleware.R OperationName: "CopySnapshot", } } + +func (c *PresignClient) PresignCopySnapshot(ctx context.Context, params *CopySnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { + if params == nil { + params = &CopySnapshotInput{} + } + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + if presignOptions.Presigner != nil { + c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + } + clientOptFns := presignOptions.ClientOptions + clientOptFns = append(clientOptFns, func(o *Options) { + o.HTTPClient = &smithyhttp.NopClient{} + }) + ctx = presignedurlcust.WithIsPresigning(ctx) + result, _, err := c.client.invokeOperation(ctx, "CopySnapshot", params, clientOptFns, + addOperationCopySnapshotMiddlewares, + c.convertToPresignMiddleware, + ) + if err != nil { + return req, err + } + out := result.(*v4.PresignedHTTPRequest) + return out, nil +} diff --git a/service/rds/api_client.go b/service/rds/api_client.go index 2be8c7cae95..f0a0cd517b2 100644 --- a/service/rds/api_client.go +++ b/service/rds/api_client.go @@ -6,6 +6,7 @@ import ( "context" "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/aws-sdk-go-v2/aws/protocol/query" "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" @@ -226,3 +227,105 @@ func addRequestResponseLogging(stack *middleware.Stack, o Options) error { LogResponseWithBody: o.ClientLogMode.IsResponseWithBody(), }, middleware.After) } + +// PresignOptions represents the presign client options +type PresignOptions struct { + // ClientOptions are list of functional options to mutate client options used by + // presign client + ClientOptions []func(*Options) + // Presigner is the presigner used by the presign url client + Presigner v4.HTTPPresigner +} + +// WithPresignClientFromClientOptions is a helper utility to retrieve a function +// that takes PresignOption as input +func WithPresignClientFromClientOptions(optFns ...func(*Options)) func(*PresignOptions) { + return withPresignClientFromClientOptions(optFns).options +} + +type withPresignClientFromClientOptions []func(*Options) + +func (w withPresignClientFromClientOptions) options(o *PresignOptions) { + o.ClientOptions = append(o.ClientOptions, w...) +} + +// PresignClient represents the presign url client +type PresignClient struct { + client *Client + presigner v4.HTTPPresigner +} + +// NewPresignClient generates a presign client using provided Client options and +// presign options +func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := New(options, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + } +} + +// NewPresignClientWrapper generates a presign client using provided API Client and +// presign options +func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := copyAPIClient(c, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + } +} + +// NewPresignClientFromConfig generates a presign client using provided AWS config +// and presign options +func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := NewFromConfig(cfg, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + } +} +func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { + return New(c.options, optFns...) +} +func (c *PresignClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { + stack.Finalize.Clear() + stack.Deserialize.Clear() + stack.Build.Remove((*awsmiddleware.ClientRequestID)(nil).ID()) + err = stack.Finalize.Add(v4.NewPresignHTTPRequestMiddleware(options.Credentials, c.presigner), middleware.After) + if err != nil { + return err + } + // convert request to a GET request + err = query.AddAsGetRequestMiddleware(stack) + if err != nil { + return err + } + return nil +} diff --git a/service/rds/api_op_CopyDBClusterSnapshot.go b/service/rds/api_op_CopyDBClusterSnapshot.go index 441daece3b4..68bc498ba04 100644 --- a/service/rds/api_op_CopyDBClusterSnapshot.go +++ b/service/rds/api_op_CopyDBClusterSnapshot.go @@ -6,13 +6,11 @@ import ( "context" "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" - "github.com/aws/aws-sdk-go-v2/aws/protocol/query" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" presignedurlcust "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url" "github.com/aws/aws-sdk-go-v2/service/rds/types" "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" - "net/http" ) // Copies a snapshot of a DB cluster. To copy a DB cluster snapshot from a shared @@ -330,52 +328,21 @@ func setCopyDBClusterSnapshotdestinationRegion(params interface{}, value string) input.destinationRegion = &value return nil } - -type copyDBClusterSnapshotHTTPPresignURLClient struct { - client *Client - presigner *v4.Signer -} - -func newCopyDBClusterSnapshotHTTPPresignURLClient(options Options, optFns ...func(*Options)) *copyDBClusterSnapshotHTTPPresignURLClient { - return ©DBClusterSnapshotHTTPPresignURLClient{ - client: New(options, optFns...), - presigner: v4.NewSigner(), +func presignCopyDBClusterSnapshot(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { + input, ok := params.(*CopyDBClusterSnapshotInput) + if !ok { + return req, fmt.Errorf("expect *CopyDBClusterSnapshotInput type, got %T", params) } -} -func (c *copyDBClusterSnapshotHTTPPresignURLClient) PresignCopyDBClusterSnapshot(ctx context.Context, params *CopyDBClusterSnapshotInput, optFns ...func(*Options)) (string, http.Header, error) { - if params == nil { - params = &CopyDBClusterSnapshotInput{} + c, ok := client.(*PresignClient) + if !ok { + return req, fmt.Errorf("expect *PresignClient type, got %T", client) } - - optFns = append(optFns, func(o *Options) { - o.HTTPClient = &smithyhttp.NopClient{} - }) - - ctx = presignedurlcust.WithIsPresigning(ctx) - result, _, err := c.client.invokeOperation(ctx, "CopyDBClusterSnapshot", params, optFns, - addOperationCopyDBClusterSnapshotMiddlewares, - c.convertToPresignMiddleware, - ) - if err != nil { - return ``, nil, err + optFn := func(o *Options) { + o.Region = region + o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } - - out := result.(*v4.PresignedHTTPRequest) - return out.URL, out.SignedHeader, nil -} -func (c *copyDBClusterSnapshotHTTPPresignURLClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { - stack.Finalize.Clear() - stack.Deserialize.Clear() - stack.Build.Remove((*awsmiddleware.ClientRequestID)(nil).ID()) - err = stack.Finalize.Add(v4.NewPresignHTTPRequestMiddleware(options.Credentials, c.presigner), middleware.After) - if err != nil { - return err - } - err = query.AddAsGetRequestMiddleware(stack) - if err != nil { - return err - } - return nil + presignOptFn := WithPresignClientFromClientOptions(optFn) + return c.PresignCopyDBClusterSnapshot(ctx, input, presignOptFn) } func addCopyDBClusterSnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ @@ -385,27 +352,12 @@ func addCopyDBClusterSnapshotPresignURLMiddleware(stack *middleware.Stack, optio CopyInput: copyCopyDBClusterSnapshotInputForPresign, SetDestinationRegion: setCopyDBClusterSnapshotdestinationRegion, SetPresignedURL: setCopyDBClusterSnapshotPreSignedUrl, + PresignOperation: presignCopyDBClusterSnapshot, }, - Presigner: &presignAutoFillCopyDBClusterSnapshotClient{client: newCopyDBClusterSnapshotHTTPPresignURLClient(options)}, + PresignClient: NewPresignClient(options), }) } -type presignAutoFillCopyDBClusterSnapshotClient struct { - client *copyDBClusterSnapshotHTTPPresignURLClient -} - -func (c *presignAutoFillCopyDBClusterSnapshotClient) PresignURL(ctx context.Context, region string, params interface{}) (string, http.Header, error) { - input, ok := params.(*CopyDBClusterSnapshotInput) - if !ok { - return ``, nil, fmt.Errorf("expect *CopyDBClusterSnapshotInput type, got %T", params) - } - optFn := func(o *Options) { - o.Region = region - o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) - } - return c.client.PresignCopyDBClusterSnapshot(ctx, input, optFn) -} - func newServiceMetadataMiddleware_opCopyDBClusterSnapshot(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, @@ -414,3 +366,30 @@ func newServiceMetadataMiddleware_opCopyDBClusterSnapshot(region string) *awsmid OperationName: "CopyDBClusterSnapshot", } } + +func (c *PresignClient) PresignCopyDBClusterSnapshot(ctx context.Context, params *CopyDBClusterSnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { + if params == nil { + params = &CopyDBClusterSnapshotInput{} + } + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + if presignOptions.Presigner != nil { + c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + } + clientOptFns := presignOptions.ClientOptions + clientOptFns = append(clientOptFns, func(o *Options) { + o.HTTPClient = &smithyhttp.NopClient{} + }) + ctx = presignedurlcust.WithIsPresigning(ctx) + result, _, err := c.client.invokeOperation(ctx, "CopyDBClusterSnapshot", params, clientOptFns, + addOperationCopyDBClusterSnapshotMiddlewares, + c.convertToPresignMiddleware, + ) + if err != nil { + return req, err + } + out := result.(*v4.PresignedHTTPRequest) + return out, nil +} diff --git a/service/rds/api_op_CopyDBSnapshot.go b/service/rds/api_op_CopyDBSnapshot.go index 8f1fe711d46..2b26a782b3f 100644 --- a/service/rds/api_op_CopyDBSnapshot.go +++ b/service/rds/api_op_CopyDBSnapshot.go @@ -6,13 +6,11 @@ import ( "context" "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" - "github.com/aws/aws-sdk-go-v2/aws/protocol/query" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" presignedurlcust "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url" "github.com/aws/aws-sdk-go-v2/service/rds/types" "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" - "net/http" ) // Copies the specified DB snapshot. The source DB snapshot must be in the @@ -283,52 +281,21 @@ func setCopyDBSnapshotdestinationRegion(params interface{}, value string) error input.destinationRegion = &value return nil } - -type copyDBSnapshotHTTPPresignURLClient struct { - client *Client - presigner *v4.Signer -} - -func newCopyDBSnapshotHTTPPresignURLClient(options Options, optFns ...func(*Options)) *copyDBSnapshotHTTPPresignURLClient { - return ©DBSnapshotHTTPPresignURLClient{ - client: New(options, optFns...), - presigner: v4.NewSigner(), +func presignCopyDBSnapshot(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { + input, ok := params.(*CopyDBSnapshotInput) + if !ok { + return req, fmt.Errorf("expect *CopyDBSnapshotInput type, got %T", params) } -} -func (c *copyDBSnapshotHTTPPresignURLClient) PresignCopyDBSnapshot(ctx context.Context, params *CopyDBSnapshotInput, optFns ...func(*Options)) (string, http.Header, error) { - if params == nil { - params = &CopyDBSnapshotInput{} + c, ok := client.(*PresignClient) + if !ok { + return req, fmt.Errorf("expect *PresignClient type, got %T", client) } - - optFns = append(optFns, func(o *Options) { - o.HTTPClient = &smithyhttp.NopClient{} - }) - - ctx = presignedurlcust.WithIsPresigning(ctx) - result, _, err := c.client.invokeOperation(ctx, "CopyDBSnapshot", params, optFns, - addOperationCopyDBSnapshotMiddlewares, - c.convertToPresignMiddleware, - ) - if err != nil { - return ``, nil, err + optFn := func(o *Options) { + o.Region = region + o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } - - out := result.(*v4.PresignedHTTPRequest) - return out.URL, out.SignedHeader, nil -} -func (c *copyDBSnapshotHTTPPresignURLClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { - stack.Finalize.Clear() - stack.Deserialize.Clear() - stack.Build.Remove((*awsmiddleware.ClientRequestID)(nil).ID()) - err = stack.Finalize.Add(v4.NewPresignHTTPRequestMiddleware(options.Credentials, c.presigner), middleware.After) - if err != nil { - return err - } - err = query.AddAsGetRequestMiddleware(stack) - if err != nil { - return err - } - return nil + presignOptFn := WithPresignClientFromClientOptions(optFn) + return c.PresignCopyDBSnapshot(ctx, input, presignOptFn) } func addCopyDBSnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ @@ -338,27 +305,12 @@ func addCopyDBSnapshotPresignURLMiddleware(stack *middleware.Stack, options Opti CopyInput: copyCopyDBSnapshotInputForPresign, SetDestinationRegion: setCopyDBSnapshotdestinationRegion, SetPresignedURL: setCopyDBSnapshotPreSignedUrl, + PresignOperation: presignCopyDBSnapshot, }, - Presigner: &presignAutoFillCopyDBSnapshotClient{client: newCopyDBSnapshotHTTPPresignURLClient(options)}, + PresignClient: NewPresignClient(options), }) } -type presignAutoFillCopyDBSnapshotClient struct { - client *copyDBSnapshotHTTPPresignURLClient -} - -func (c *presignAutoFillCopyDBSnapshotClient) PresignURL(ctx context.Context, region string, params interface{}) (string, http.Header, error) { - input, ok := params.(*CopyDBSnapshotInput) - if !ok { - return ``, nil, fmt.Errorf("expect *CopyDBSnapshotInput type, got %T", params) - } - optFn := func(o *Options) { - o.Region = region - o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) - } - return c.client.PresignCopyDBSnapshot(ctx, input, optFn) -} - func newServiceMetadataMiddleware_opCopyDBSnapshot(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, @@ -367,3 +319,30 @@ func newServiceMetadataMiddleware_opCopyDBSnapshot(region string) *awsmiddleware OperationName: "CopyDBSnapshot", } } + +func (c *PresignClient) PresignCopyDBSnapshot(ctx context.Context, params *CopyDBSnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { + if params == nil { + params = &CopyDBSnapshotInput{} + } + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + if presignOptions.Presigner != nil { + c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + } + clientOptFns := presignOptions.ClientOptions + clientOptFns = append(clientOptFns, func(o *Options) { + o.HTTPClient = &smithyhttp.NopClient{} + }) + ctx = presignedurlcust.WithIsPresigning(ctx) + result, _, err := c.client.invokeOperation(ctx, "CopyDBSnapshot", params, clientOptFns, + addOperationCopyDBSnapshotMiddlewares, + c.convertToPresignMiddleware, + ) + if err != nil { + return req, err + } + out := result.(*v4.PresignedHTTPRequest) + return out, nil +} diff --git a/service/rds/api_op_CreateDBCluster.go b/service/rds/api_op_CreateDBCluster.go index 85d3f03016d..d216973572d 100644 --- a/service/rds/api_op_CreateDBCluster.go +++ b/service/rds/api_op_CreateDBCluster.go @@ -6,13 +6,11 @@ import ( "context" "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" - "github.com/aws/aws-sdk-go-v2/aws/protocol/query" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" presignedurlcust "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url" "github.com/aws/aws-sdk-go-v2/service/rds/types" "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" - "net/http" ) // Creates a new Amazon Aurora DB cluster. You can use the @@ -467,52 +465,21 @@ func setCreateDBClusterdestinationRegion(params interface{}, value string) error input.destinationRegion = &value return nil } - -type createDBClusterHTTPPresignURLClient struct { - client *Client - presigner *v4.Signer -} - -func newCreateDBClusterHTTPPresignURLClient(options Options, optFns ...func(*Options)) *createDBClusterHTTPPresignURLClient { - return &createDBClusterHTTPPresignURLClient{ - client: New(options, optFns...), - presigner: v4.NewSigner(), +func presignCreateDBCluster(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { + input, ok := params.(*CreateDBClusterInput) + if !ok { + return req, fmt.Errorf("expect *CreateDBClusterInput type, got %T", params) } -} -func (c *createDBClusterHTTPPresignURLClient) PresignCreateDBCluster(ctx context.Context, params *CreateDBClusterInput, optFns ...func(*Options)) (string, http.Header, error) { - if params == nil { - params = &CreateDBClusterInput{} + c, ok := client.(*PresignClient) + if !ok { + return req, fmt.Errorf("expect *PresignClient type, got %T", client) } - - optFns = append(optFns, func(o *Options) { - o.HTTPClient = &smithyhttp.NopClient{} - }) - - ctx = presignedurlcust.WithIsPresigning(ctx) - result, _, err := c.client.invokeOperation(ctx, "CreateDBCluster", params, optFns, - addOperationCreateDBClusterMiddlewares, - c.convertToPresignMiddleware, - ) - if err != nil { - return ``, nil, err + optFn := func(o *Options) { + o.Region = region + o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } - - out := result.(*v4.PresignedHTTPRequest) - return out.URL, out.SignedHeader, nil -} -func (c *createDBClusterHTTPPresignURLClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { - stack.Finalize.Clear() - stack.Deserialize.Clear() - stack.Build.Remove((*awsmiddleware.ClientRequestID)(nil).ID()) - err = stack.Finalize.Add(v4.NewPresignHTTPRequestMiddleware(options.Credentials, c.presigner), middleware.After) - if err != nil { - return err - } - err = query.AddAsGetRequestMiddleware(stack) - if err != nil { - return err - } - return nil + presignOptFn := WithPresignClientFromClientOptions(optFn) + return c.PresignCreateDBCluster(ctx, input, presignOptFn) } func addCreateDBClusterPresignURLMiddleware(stack *middleware.Stack, options Options) error { return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ @@ -522,27 +489,12 @@ func addCreateDBClusterPresignURLMiddleware(stack *middleware.Stack, options Opt CopyInput: copyCreateDBClusterInputForPresign, SetDestinationRegion: setCreateDBClusterdestinationRegion, SetPresignedURL: setCreateDBClusterPreSignedUrl, + PresignOperation: presignCreateDBCluster, }, - Presigner: &presignAutoFillCreateDBClusterClient{client: newCreateDBClusterHTTPPresignURLClient(options)}, + PresignClient: NewPresignClient(options), }) } -type presignAutoFillCreateDBClusterClient struct { - client *createDBClusterHTTPPresignURLClient -} - -func (c *presignAutoFillCreateDBClusterClient) PresignURL(ctx context.Context, region string, params interface{}) (string, http.Header, error) { - input, ok := params.(*CreateDBClusterInput) - if !ok { - return ``, nil, fmt.Errorf("expect *CreateDBClusterInput type, got %T", params) - } - optFn := func(o *Options) { - o.Region = region - o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) - } - return c.client.PresignCreateDBCluster(ctx, input, optFn) -} - func newServiceMetadataMiddleware_opCreateDBCluster(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, @@ -551,3 +503,30 @@ func newServiceMetadataMiddleware_opCreateDBCluster(region string) *awsmiddlewar OperationName: "CreateDBCluster", } } + +func (c *PresignClient) PresignCreateDBCluster(ctx context.Context, params *CreateDBClusterInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { + if params == nil { + params = &CreateDBClusterInput{} + } + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + if presignOptions.Presigner != nil { + c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + } + clientOptFns := presignOptions.ClientOptions + clientOptFns = append(clientOptFns, func(o *Options) { + o.HTTPClient = &smithyhttp.NopClient{} + }) + ctx = presignedurlcust.WithIsPresigning(ctx) + result, _, err := c.client.invokeOperation(ctx, "CreateDBCluster", params, clientOptFns, + addOperationCreateDBClusterMiddlewares, + c.convertToPresignMiddleware, + ) + if err != nil { + return req, err + } + out := result.(*v4.PresignedHTTPRequest) + return out, nil +} diff --git a/service/rds/api_op_CreateDBInstanceReadReplica.go b/service/rds/api_op_CreateDBInstanceReadReplica.go index b995932f4c6..7a9a5589f25 100644 --- a/service/rds/api_op_CreateDBInstanceReadReplica.go +++ b/service/rds/api_op_CreateDBInstanceReadReplica.go @@ -6,13 +6,11 @@ import ( "context" "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" - "github.com/aws/aws-sdk-go-v2/aws/protocol/query" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" presignedurlcust "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url" "github.com/aws/aws-sdk-go-v2/service/rds/types" "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" - "net/http" ) // Creates a new DB instance that acts as a read replica for an existing source DB @@ -477,52 +475,21 @@ func setCreateDBInstanceReadReplicadestinationRegion(params interface{}, value s input.destinationRegion = &value return nil } - -type createDBInstanceReadReplicaHTTPPresignURLClient struct { - client *Client - presigner *v4.Signer -} - -func newCreateDBInstanceReadReplicaHTTPPresignURLClient(options Options, optFns ...func(*Options)) *createDBInstanceReadReplicaHTTPPresignURLClient { - return &createDBInstanceReadReplicaHTTPPresignURLClient{ - client: New(options, optFns...), - presigner: v4.NewSigner(), +func presignCreateDBInstanceReadReplica(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { + input, ok := params.(*CreateDBInstanceReadReplicaInput) + if !ok { + return req, fmt.Errorf("expect *CreateDBInstanceReadReplicaInput type, got %T", params) } -} -func (c *createDBInstanceReadReplicaHTTPPresignURLClient) PresignCreateDBInstanceReadReplica(ctx context.Context, params *CreateDBInstanceReadReplicaInput, optFns ...func(*Options)) (string, http.Header, error) { - if params == nil { - params = &CreateDBInstanceReadReplicaInput{} + c, ok := client.(*PresignClient) + if !ok { + return req, fmt.Errorf("expect *PresignClient type, got %T", client) } - - optFns = append(optFns, func(o *Options) { - o.HTTPClient = &smithyhttp.NopClient{} - }) - - ctx = presignedurlcust.WithIsPresigning(ctx) - result, _, err := c.client.invokeOperation(ctx, "CreateDBInstanceReadReplica", params, optFns, - addOperationCreateDBInstanceReadReplicaMiddlewares, - c.convertToPresignMiddleware, - ) - if err != nil { - return ``, nil, err + optFn := func(o *Options) { + o.Region = region + o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } - - out := result.(*v4.PresignedHTTPRequest) - return out.URL, out.SignedHeader, nil -} -func (c *createDBInstanceReadReplicaHTTPPresignURLClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { - stack.Finalize.Clear() - stack.Deserialize.Clear() - stack.Build.Remove((*awsmiddleware.ClientRequestID)(nil).ID()) - err = stack.Finalize.Add(v4.NewPresignHTTPRequestMiddleware(options.Credentials, c.presigner), middleware.After) - if err != nil { - return err - } - err = query.AddAsGetRequestMiddleware(stack) - if err != nil { - return err - } - return nil + presignOptFn := WithPresignClientFromClientOptions(optFn) + return c.PresignCreateDBInstanceReadReplica(ctx, input, presignOptFn) } func addCreateDBInstanceReadReplicaPresignURLMiddleware(stack *middleware.Stack, options Options) error { return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ @@ -532,27 +499,12 @@ func addCreateDBInstanceReadReplicaPresignURLMiddleware(stack *middleware.Stack, CopyInput: copyCreateDBInstanceReadReplicaInputForPresign, SetDestinationRegion: setCreateDBInstanceReadReplicadestinationRegion, SetPresignedURL: setCreateDBInstanceReadReplicaPreSignedUrl, + PresignOperation: presignCreateDBInstanceReadReplica, }, - Presigner: &presignAutoFillCreateDBInstanceReadReplicaClient{client: newCreateDBInstanceReadReplicaHTTPPresignURLClient(options)}, + PresignClient: NewPresignClient(options), }) } -type presignAutoFillCreateDBInstanceReadReplicaClient struct { - client *createDBInstanceReadReplicaHTTPPresignURLClient -} - -func (c *presignAutoFillCreateDBInstanceReadReplicaClient) PresignURL(ctx context.Context, region string, params interface{}) (string, http.Header, error) { - input, ok := params.(*CreateDBInstanceReadReplicaInput) - if !ok { - return ``, nil, fmt.Errorf("expect *CreateDBInstanceReadReplicaInput type, got %T", params) - } - optFn := func(o *Options) { - o.Region = region - o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) - } - return c.client.PresignCreateDBInstanceReadReplica(ctx, input, optFn) -} - func newServiceMetadataMiddleware_opCreateDBInstanceReadReplica(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, @@ -561,3 +513,30 @@ func newServiceMetadataMiddleware_opCreateDBInstanceReadReplica(region string) * OperationName: "CreateDBInstanceReadReplica", } } + +func (c *PresignClient) PresignCreateDBInstanceReadReplica(ctx context.Context, params *CreateDBInstanceReadReplicaInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { + if params == nil { + params = &CreateDBInstanceReadReplicaInput{} + } + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + if presignOptions.Presigner != nil { + c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + } + clientOptFns := presignOptions.ClientOptions + clientOptFns = append(clientOptFns, func(o *Options) { + o.HTTPClient = &smithyhttp.NopClient{} + }) + ctx = presignedurlcust.WithIsPresigning(ctx) + result, _, err := c.client.invokeOperation(ctx, "CreateDBInstanceReadReplica", params, clientOptFns, + addOperationCreateDBInstanceReadReplicaMiddlewares, + c.convertToPresignMiddleware, + ) + if err != nil { + return req, err + } + out := result.(*v4.PresignedHTTPRequest) + return out, nil +} diff --git a/service/s3/api_client.go b/service/s3/api_client.go index 2825f4011d6..3f59080fb8c 100644 --- a/service/s3/api_client.go +++ b/service/s3/api_client.go @@ -258,3 +258,125 @@ func addRequestResponseLogging(stack *middleware.Stack, o Options) error { LogResponseWithBody: o.ClientLogMode.IsResponseWithBody(), }, middleware.After) } + +// PresignOptions represents the presign client options +type PresignOptions struct { + // ClientOptions are list of functional options to mutate client options used by + // presign client + ClientOptions []func(*Options) + // Presigner is the presigner used by the presign url client + Presigner v4.HTTPPresigner + // Expires sets the expiration duration for the generated presign url + Expires time.Duration +} + +// WithPresignClientFromClientOptions is a helper utility to retrieve a function +// that takes PresignOption as input +func WithPresignClientFromClientOptions(optFns ...func(*Options)) func(*PresignOptions) { + return withPresignClientFromClientOptions(optFns).options +} + +type withPresignClientFromClientOptions []func(*Options) + +func (w withPresignClientFromClientOptions) options(o *PresignOptions) { + o.ClientOptions = append(o.ClientOptions, w...) +} + +// WithPresignExpires is a helper utility to append Expires value on presign +// options optional function +func WithPresignExpires(dur time.Duration) func(*PresignOptions) { + return withPresignExpires(dur).options +} + +type withPresignExpires time.Duration + +func (w withPresignExpires) options(o *PresignOptions) { + o.Expires = time.Duration(w) +} + +// PresignClient represents the presign url client +type PresignClient struct { + client *Client + presigner v4.HTTPPresigner + expires time.Duration +} + +// NewPresignClient generates a presign client using provided Client options and +// presign options +func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := New(options, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + expires: presignOptions.Expires, + } +} + +// NewPresignClientWrapper generates a presign client using provided API Client and +// presign options +func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := copyAPIClient(c, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + expires: presignOptions.Expires, + } +} + +// NewPresignClientFromConfig generates a presign client using provided AWS config +// and presign options +func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) *PresignClient { + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + client := NewFromConfig(cfg, presignOptions.ClientOptions...) + var presigner v4.HTTPPresigner + if presignOptions.Presigner != nil { + presigner = presignOptions.Presigner + } else { + presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, presigner: presigner, + expires: presignOptions.Expires, + } +} +func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { + return New(c.options, optFns...) +} +func (c *PresignClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { + stack.Finalize.Clear() + stack.Deserialize.Clear() + stack.Build.Remove((*awsmiddleware.ClientRequestID)(nil).ID()) + err = stack.Finalize.Add(v4.NewPresignHTTPRequestMiddleware(options.Credentials, c.presigner), middleware.After) + if err != nil { + return err + } + if c.expires != 0 { + // add middleware to set expiration for s3 presigned url + err = stack.Build.Add(&s3cust.AddExpiresOnPresignedURL{Expires: c.expires}, middleware.After) + if err != nil { + return err + } + } + return nil +} diff --git a/service/s3/api_op_PutObject.go b/service/s3/api_op_PutObject.go index 65a55f775ee..e66bb9a73dd 100644 --- a/service/s3/api_op_PutObject.go +++ b/service/s3/api_op_PutObject.go @@ -7,6 +7,7 @@ import ( awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" s3cust "github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations" + presignedurlcust "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" @@ -424,3 +425,30 @@ func addPutObjectUpdateEndpoint(stack *middleware.Stack, options Options) error UseARNRegion: options.UseARNRegion, }) } + +func (c *PresignClient) PresignPutObject(ctx context.Context, params *PutObjectInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { + if params == nil { + params = &PutObjectInput{} + } + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + if presignOptions.Presigner != nil { + c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + } + clientOptFns := presignOptions.ClientOptions + clientOptFns = append(clientOptFns, func(o *Options) { + o.HTTPClient = &smithyhttp.NopClient{} + }) + ctx = presignedurlcust.WithIsPresigning(ctx) + result, _, err := c.client.invokeOperation(ctx, "PutObject", params, clientOptFns, + addOperationPutObjectMiddlewares, + c.convertToPresignMiddleware, + ) + if err != nil { + return req, err + } + out := result.(*v4.PresignedHTTPRequest) + return out, nil +} diff --git a/service/s3/go.mod b/service/s3/go.mod index 5882d405dd2..08f15b1aa29 100644 --- a/service/s3/go.mod +++ b/service/s3/go.mod @@ -6,6 +6,7 @@ require ( github.com/aws/aws-sdk-go-v2 v0.29.1-0.20201113222241-726e4a15683d github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v0.3.1-0.20201113222241-726e4a15683d github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1-0.20201113222241-726e4a15683d + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.0 github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4 ) @@ -13,4 +14,6 @@ replace github.com/aws/aws-sdk-go-v2 => ../../ replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../service/internal/accept-encoding/ +replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../service/internal/presigned-url/ + replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../service/internal/s3shared/ diff --git a/service/s3/go.sum b/service/s3/go.sum index 57ffc83dc1c..e1dc3ec43bf 100644 --- a/service/s3/go.sum +++ b/service/s3/go.sum @@ -5,6 +5,8 @@ github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4/go.mod h1:hPOQ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/service/s3/internal/configtesting/go.mod b/service/s3/internal/configtesting/go.mod index 306d989e686..7e5f425a344 100644 --- a/service/s3/internal/configtesting/go.mod +++ b/service/s3/internal/configtesting/go.mod @@ -22,3 +22,5 @@ replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../../se replace github.com/aws/aws-sdk-go-v2/service/sts => ../../../../service/sts/ replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../../service/internal/accept-encoding/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../../service/internal/presigned-url/ diff --git a/service/s3/internal/configtesting/go.sum b/service/s3/internal/configtesting/go.sum index 57ffc83dc1c..e1dc3ec43bf 100644 --- a/service/s3/internal/configtesting/go.sum +++ b/service/s3/internal/configtesting/go.sum @@ -5,6 +5,8 @@ github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4/go.mod h1:hPOQ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= From 9817fbc48269900b323960cfd738cfe47c081661 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 11 Nov 2020 02:10:21 -0800 Subject: [PATCH 08/24] go mod changes for pkgs that depend on s3 and now presigned url pkgs --- example/service/s3/listObjects/go.mod | 2 ++ example/service/s3/listObjects/go.sum | 2 ++ feature/s3/manager/go.mod | 4 +++- feature/s3/manager/go.sum | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/example/service/s3/listObjects/go.mod b/example/service/s3/listObjects/go.mod index 3bad0eece6d..e9f2a4f3273 100644 --- a/example/service/s3/listObjects/go.mod +++ b/example/service/s3/listObjects/go.mod @@ -23,3 +23,5 @@ replace github.com/aws/aws-sdk-go-v2/service/sts => ../../../../service/sts/ replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../../service/internal/accept-encoding/ replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../../service/internal/s3shared/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../../service/internal/presigned-url/ diff --git a/example/service/s3/listObjects/go.sum b/example/service/s3/listObjects/go.sum index 57ffc83dc1c..e1dc3ec43bf 100644 --- a/example/service/s3/listObjects/go.sum +++ b/example/service/s3/listObjects/go.sum @@ -5,6 +5,8 @@ github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4/go.mod h1:hPOQ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/feature/s3/manager/go.mod b/feature/s3/manager/go.mod index 45c728cce68..db589b6bbca 100644 --- a/feature/s3/manager/go.mod +++ b/feature/s3/manager/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v0.29.0 github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4 - github.com/google/go-cmp v0.4.1 + github.com/google/go-cmp v0.5.2 ) replace github.com/aws/aws-sdk-go-v2 => ../../../ @@ -26,3 +26,5 @@ replace github.com/aws/aws-sdk-go-v2/service/sts => ../../../service/sts/ replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../service/internal/accept-encoding/ replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../service/internal/s3shared/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../service/internal/presigned-url/ diff --git a/feature/s3/manager/go.sum b/feature/s3/manager/go.sum index 2f07ad94cff..75ff214e0e0 100644 --- a/feature/s3/manager/go.sum +++ b/feature/s3/manager/go.sum @@ -6,6 +6,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= From 2c1a79b1e0f881dc0963f071c745799c0b0d661c Mon Sep 17 00:00:00 2001 From: skotambkar Date: Sun, 15 Nov 2020 22:58:36 -0800 Subject: [PATCH 09/24] feedback changes --- .../AwsHttpPresignURLClientGenerator.java | 102 ++++++++++++------ .../internal/presigned-url/middleware_test.go | 2 +- .../customizations/presigned_expires.go | 4 +- 3 files changed, 74 insertions(+), 34 deletions(-) diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java index f0513f12f41..b79babeccfe 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java @@ -47,6 +47,7 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration { private static final String PRESIGN_CLIENT = "PresignClient"; private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true); + private static final String NEW_CLIENT_INTERNAL = "newPresignClient"; private static final String NEW_CLIENT = "NewPresignClient"; private static final String NEW_CLIENT_FROM_SERVICE = "NewPresignClientWrapper"; private static final String NEW_CLIENT_FROM_CONFIG = "NewPresignClientFromConfig"; @@ -57,9 +58,8 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration { private static final String PRESIGN_OPTIONS_FROM_CLIENT_OPTIONS = "WithPresignClientFromClientOptions"; private static final String PRESIGN_OPTIONS_FROM_EXPIRES = "WithPresignExpires"; - private static final String PRESIGN_SIGNER = "Presigner"; private static final Symbol presignerInterfaceSymbol = SymbolUtils.createPointableSymbolBuilder( - "HTTPPresigner", AwsGoDependency.AWS_SIGNER_V4 + "HTTPPresignerV4" ).build(); private static final Symbol v4NewPresignerSymbol = SymbolUtils.createPointableSymbolBuilder( "NewSigner", AwsGoDependency.AWS_SIGNER_V4 @@ -126,19 +126,18 @@ private final void returnPresignClientConstructor( Model model, ServiceShape serviceShape ) { - writer.write("var presigner $T", presignerInterfaceSymbol); - writer.openBlock("if presignOptions.Presigner != nil {", "} else {", () -> { - writer.write("presigner = presignOptions.Presigner"); - }).write("presigner = $T() }", v4NewPresignerSymbol).insertTrailingNewline(); + writer.openBlock("if presignOptions.Presigner == nil {", "}", () -> { + writer.write("presignOptions.Presigner = $T()", v4NewPresignerSymbol); + }); writer.openBlock("return &$L{", "}", presignClientSymbol, () -> { - writer.write("client: client, presigner: presigner,"); + writer.write("client: client,"); + writer.write("presigner: presignOptions.Presigner,"); // if s3 assign expires value on client if (isS3ServiceShape(model, serviceShape)) { writer.write("expires: presignOptions.Expires,"); } }); - } @Override @@ -156,7 +155,6 @@ public void writeAdditionalFiles( GoDelegator goDelegator ) { ServiceShape serviceShape = settings.getService(model); - if (!PRESIGNER_MAP.containsKey(serviceShape.getId())) { return; } @@ -168,6 +166,9 @@ public void writeAdditionalFiles( // delegator for service shape goDelegator.useShapeWriter(serviceShape, (writer) -> { + // generate presigner interface + writePresignInterface(writer, model, symbolProvider, serviceShape); + // generate presign options and helpers per service writePresignOptionType(writer, model, symbolProvider, serviceShape); @@ -181,7 +182,6 @@ public void writeAdditionalFiles( writeConvertToPresignMiddleware(writer, model, symbolProvider, serviceShape); }); - for (ShapeId operationId : serviceShape.getAllOperations()) { OperationShape operationShape = model.expectShape(operationId, OperationShape.class); if (!validOperations.contains(operationShape.getId())) { @@ -207,6 +207,10 @@ private void writePresignOperationFunction( Shape operationInputShape = model.expectShape(operationShape.getInput().get()); Symbol operationInputSymbol = symbolProvider.toSymbol(operationInputShape); + writer.writeDocs( + String.format("Presign%s is used to generate a presigned HTTP Request which contains presigned URL, signed headers " + + "and HTTP method used.", operationSymbol.getName()) + ); writer.openBlock( "func (c *$T) Presign$T(ctx context.Context, params $P, optFns ...func($P)) " + "(req *v4.PresignedHTTPRequest, err error) {", @@ -343,24 +347,34 @@ private void writePresignClientType( writer.addUseImports(SmithyGoDependency.CONTEXT); writer.addUseImports(AwsGoDependency.AWS_SIGNER_V4); - - Symbol presignerInterfaceSymbol = SymbolUtils.createPointableSymbolBuilder( - "HTTPPresigner", AwsGoDependency.AWS_SIGNER_V4 - ).build(); - writer.writeDocs(String.format("%s represents the presign url client", PRESIGN_CLIENT)); writer.openBlock("type $T struct {", "}", presignClientSymbol, () -> { writer.write("client *Client"); - writer.write("presigner v4.HTTPPresigner"); + writer.write("presigner $T", presignerInterfaceSymbol); if (isS3ServiceShape(model, serviceShape)) { writer.addUseImports(SmithyGoDependency.TIME); writer.write("expires time.Duration"); } - }); - // generate constructors + // generate internal constructors + writer.openBlock("func $L(client *Client, options $T) $P {", "}", + NEW_CLIENT_INTERNAL, presignOptionsSymbol, presignClientSymbol, () -> { + writer.openBlock("if options.Presigner == nil {", "}", () -> { + writer.write("options.Presigner = $T()", v4NewPresignerSymbol); + }); + + writer.openBlock("return &$L{", "}", presignClientSymbol, () -> { + writer.write("client: client,"); + writer.write("presigner: options.Presigner,"); + // if s3 assign expires value on client + if (isS3ServiceShape(model, serviceShape)) { + writer.write("expires: options.Expires,"); + } + }); + }); + writer.write(""); // generate NewPresignClient writer.writeDocs( @@ -371,11 +385,10 @@ private void writePresignClientType( NEW_CLIENT, presignOptionsSymbol, presignClientSymbol, () -> { processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); writer.insertTrailingNewline(); - writer.write("client := New(options, presignOptions.ClientOptions...)").insertTrailingNewline(); writer.insertTrailingNewline(); - returnPresignClientConstructor(writer, model, serviceShape); + writer.write("return $L(client, presignOptions)", NEW_CLIENT_INTERNAL); }).insertTrailingNewline(); // generate NewPresignClientWrapper @@ -387,11 +400,10 @@ private void writePresignClientType( NEW_CLIENT_FROM_SERVICE, presignOptionsSymbol, presignClientSymbol, () -> { processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); writer.insertTrailingNewline(); - writer.write("client := copyAPIClient(c, presignOptions.ClientOptions...)"); writer.insertTrailingNewline(); - returnPresignClientConstructor(writer, model, serviceShape); + writer.write("return $L(client, presignOptions)", NEW_CLIENT_INTERNAL); }).insertTrailingNewline(); // generate NewPresignClientFromConfig @@ -403,14 +415,13 @@ private void writePresignClientType( NEW_CLIENT_FROM_CONFIG, presignOptionsSymbol, presignClientSymbol, () -> { processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); writer.insertTrailingNewline(); - writer.write("client := NewFromConfig(cfg, presignOptions.ClientOptions...)"); writer.insertTrailingNewline(); - returnPresignClientConstructor(writer, model, serviceShape); + writer.write("return $L(client, presignOptions)", NEW_CLIENT_INTERNAL); }).insertTrailingNewline(); - writer.insertTrailingNewline(); + writer.write(""); } /** @@ -430,8 +441,30 @@ private void writePresignClientHelpers( COPY_API_CLIENT, () -> { writer.write("return New(c.options, optFns...)"); writer.insertTrailingNewline(); - }).insertTrailingNewline(); - writer.insertTrailingNewline(); + }); + writer.write(""); + } + + /** + * Writes the presigner interface used by the presign url client + */ + public void writePresignInterface( + GoWriter writer, + Model model, + SymbolProvider symbolProvider, + ServiceShape serviceShape + ) { + writer.writeDocs( + String.format("%s represents presigner interface used by presign url client", presignerInterfaceSymbol.getName()) + ); + writer.openBlock("type $T interface {", "}", presignerInterfaceSymbol, () -> { + writer.write("PresignHTTP("); + writer.write("ctx context.Context, credentials aws.Credentials, r *http.Request, "); + writer.write("payloadHash string, service string, region string, signingTime time.Time, "); + writer.write(") (url string, signedHeader http.Header, err error)"); + }); + + writer.insertTrailingNewline(); } /** @@ -446,24 +479,29 @@ public void writePresignOptionType( ServiceShape serviceShape ) { writer.addUseImports(SmithyGoDependency.CONTEXT); - writer.addUseImports(AwsGoDependency.AWS_SIGNER_V4); - Symbol presignOptionSymbol = buildSymbol(PRESIGN_OPTIONS, true); // generate presign options writer.writeDocs(String.format("%s represents the presign client options", PRESIGN_OPTIONS)); writer.openBlock("type $T struct {", "}", presignOptionSymbol, () -> { + writer.write(""); writer.writeDocs( - "ClientOptions are list of functional options to mutate client options used by presign client"); + "ClientOptions are list of functional options to mutate client options used by presign client" + ); writer.write("ClientOptions []func(*Options)"); - writer.insertTrailingNewline(); + writer.write(""); writer.writeDocs("Presigner is the presigner used by the presign url client"); writer.write("Presigner $T", presignerInterfaceSymbol); // s3 service has an additional Expires options if (isS3ServiceShape(model, serviceShape)) { - writer.writeDocs("Expires sets the expiration duration for the generated presign url"); + writer.write(""); + writer.writeDocs( + String.format("Expires sets the expiration duration for the generated presign url. This should " + + "be the duration in seconds the presigned URL should be considered valid for." + ) + ); writer.write("Expires time.Duration"); } }); diff --git a/service/internal/presigned-url/middleware_test.go b/service/internal/presigned-url/middleware_test.go index b3ce8e669a0..3af55ff4a0d 100644 --- a/service/internal/presigned-url/middleware_test.go +++ b/service/internal/presigned-url/middleware_test.go @@ -146,7 +146,7 @@ func presignURL(ctx context.Context, client interface{}, srcRegion string, param return &v4.PresignedHTTPRequest{ URL: "https://example." + srcRegion + ".amazonaws.com/?DestinationRegion=" + in.DestinationRegion, - Method: "POST", + Method: "GET", SignedHeader: http.Header{}, }, nil } diff --git a/service/s3/internal/customizations/presigned_expires.go b/service/s3/internal/customizations/presigned_expires.go index 2890e7df5d6..54121c883a1 100644 --- a/service/s3/internal/customizations/presigned_expires.go +++ b/service/s3/internal/customizations/presigned_expires.go @@ -13,7 +13,9 @@ import ( // AddExpiresOnPresignedURL represents a build middleware used to assign // expiration on a presigned URL type AddExpiresOnPresignedURL struct { - // Expires is time.Duration within which presigned url should be expired + + // Expires is time.Duration within which presigned url should be expired. + // This should be the duration in seconds the presigned URL should be considered valid for. Expires time.Duration } From b090abd557c09e1d01f18c19987a8b36bd4c1af7 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Mon, 16 Nov 2020 00:08:20 -0800 Subject: [PATCH 10/24] generated service changes --- service/ec2/api_client.go | 56 ++++++++-------- service/ec2/api_op_CopySnapshot.go | 2 + service/rds/api_client.go | 56 ++++++++-------- service/rds/api_op_CopyDBClusterSnapshot.go | 2 + service/rds/api_op_CopyDBSnapshot.go | 2 + service/rds/api_op_CreateDBCluster.go | 2 + .../rds/api_op_CreateDBInstanceReadReplica.go | 2 + service/s3/api_client.go | 64 +++++++++---------- service/s3/api_op_PutObject.go | 2 + 9 files changed, 97 insertions(+), 91 deletions(-) diff --git a/service/ec2/api_client.go b/service/ec2/api_client.go index 36ccfb94e7b..a9b71c53049 100644 --- a/service/ec2/api_client.go +++ b/service/ec2/api_client.go @@ -248,13 +248,23 @@ func addRequestResponseLogging(stack *middleware.Stack, o Options) error { }, middleware.After) } +// HTTPPresignerV4 represents presigner interface used by presign url client +type HTTPPresignerV4 interface { + PresignHTTP( + ctx context.Context, credentials aws.Credentials, r *http.Request, + payloadHash string, service string, region string, signingTime time.Time, + ) (url string, signedHeader http.Header, err error) +} + // PresignOptions represents the presign client options type PresignOptions struct { + // ClientOptions are list of functional options to mutate client options used by // presign client ClientOptions []func(*Options) + // Presigner is the presigner used by the presign url client - Presigner v4.HTTPPresigner + Presigner HTTPPresignerV4 } // WithPresignClientFromClientOptions is a helper utility to retrieve a function @@ -272,7 +282,17 @@ func (w withPresignClientFromClientOptions) options(o *PresignOptions) { // PresignClient represents the presign url client type PresignClient struct { client *Client - presigner v4.HTTPPresigner + presigner HTTPPresignerV4 +} + +func newPresignClient(client *Client, options PresignOptions) *PresignClient { + if options.Presigner == nil { + options.Presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, + presigner: options.Presigner, + } } // NewPresignClient generates a presign client using provided Client options and @@ -283,15 +303,7 @@ func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *Presign fn(&presignOptions) } client := New(options, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - } + return newPresignClient(client, presignOptions) } // NewPresignClientWrapper generates a presign client using provided API Client and @@ -302,15 +314,7 @@ func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *Presig fn(&presignOptions) } client := copyAPIClient(c, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - } + return newPresignClient(client, presignOptions) } // NewPresignClientFromConfig generates a presign client using provided AWS config @@ -321,19 +325,13 @@ func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) fn(&presignOptions) } client := NewFromConfig(cfg, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - } + return newPresignClient(client, presignOptions) } + func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { return New(c.options, optFns...) } + func (c *PresignClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { stack.Finalize.Clear() stack.Deserialize.Clear() diff --git a/service/ec2/api_op_CopySnapshot.go b/service/ec2/api_op_CopySnapshot.go index 2e38ecebd9b..d16d7d7ec79 100644 --- a/service/ec2/api_op_CopySnapshot.go +++ b/service/ec2/api_op_CopySnapshot.go @@ -277,6 +277,8 @@ func newServiceMetadataMiddleware_opCopySnapshot(region string) *awsmiddleware.R } } +// PresignCopySnapshot is used to generate a presigned HTTP Request which contains +// presigned URL, signed headers and HTTP method used. func (c *PresignClient) PresignCopySnapshot(ctx context.Context, params *CopySnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { if params == nil { params = &CopySnapshotInput{} diff --git a/service/rds/api_client.go b/service/rds/api_client.go index f0a0cd517b2..b00e75eb627 100644 --- a/service/rds/api_client.go +++ b/service/rds/api_client.go @@ -228,13 +228,23 @@ func addRequestResponseLogging(stack *middleware.Stack, o Options) error { }, middleware.After) } +// HTTPPresignerV4 represents presigner interface used by presign url client +type HTTPPresignerV4 interface { + PresignHTTP( + ctx context.Context, credentials aws.Credentials, r *http.Request, + payloadHash string, service string, region string, signingTime time.Time, + ) (url string, signedHeader http.Header, err error) +} + // PresignOptions represents the presign client options type PresignOptions struct { + // ClientOptions are list of functional options to mutate client options used by // presign client ClientOptions []func(*Options) + // Presigner is the presigner used by the presign url client - Presigner v4.HTTPPresigner + Presigner HTTPPresignerV4 } // WithPresignClientFromClientOptions is a helper utility to retrieve a function @@ -252,7 +262,17 @@ func (w withPresignClientFromClientOptions) options(o *PresignOptions) { // PresignClient represents the presign url client type PresignClient struct { client *Client - presigner v4.HTTPPresigner + presigner HTTPPresignerV4 +} + +func newPresignClient(client *Client, options PresignOptions) *PresignClient { + if options.Presigner == nil { + options.Presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, + presigner: options.Presigner, + } } // NewPresignClient generates a presign client using provided Client options and @@ -263,15 +283,7 @@ func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *Presign fn(&presignOptions) } client := New(options, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - } + return newPresignClient(client, presignOptions) } // NewPresignClientWrapper generates a presign client using provided API Client and @@ -282,15 +294,7 @@ func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *Presig fn(&presignOptions) } client := copyAPIClient(c, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - } + return newPresignClient(client, presignOptions) } // NewPresignClientFromConfig generates a presign client using provided AWS config @@ -301,19 +305,13 @@ func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) fn(&presignOptions) } client := NewFromConfig(cfg, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - } + return newPresignClient(client, presignOptions) } + func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { return New(c.options, optFns...) } + func (c *PresignClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { stack.Finalize.Clear() stack.Deserialize.Clear() diff --git a/service/rds/api_op_CopyDBClusterSnapshot.go b/service/rds/api_op_CopyDBClusterSnapshot.go index 68bc498ba04..0a03b67aad3 100644 --- a/service/rds/api_op_CopyDBClusterSnapshot.go +++ b/service/rds/api_op_CopyDBClusterSnapshot.go @@ -367,6 +367,8 @@ func newServiceMetadataMiddleware_opCopyDBClusterSnapshot(region string) *awsmid } } +// PresignCopyDBClusterSnapshot is used to generate a presigned HTTP Request which +// contains presigned URL, signed headers and HTTP method used. func (c *PresignClient) PresignCopyDBClusterSnapshot(ctx context.Context, params *CopyDBClusterSnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { if params == nil { params = &CopyDBClusterSnapshotInput{} diff --git a/service/rds/api_op_CopyDBSnapshot.go b/service/rds/api_op_CopyDBSnapshot.go index 2b26a782b3f..4e037541c07 100644 --- a/service/rds/api_op_CopyDBSnapshot.go +++ b/service/rds/api_op_CopyDBSnapshot.go @@ -320,6 +320,8 @@ func newServiceMetadataMiddleware_opCopyDBSnapshot(region string) *awsmiddleware } } +// PresignCopyDBSnapshot is used to generate a presigned HTTP Request which +// contains presigned URL, signed headers and HTTP method used. func (c *PresignClient) PresignCopyDBSnapshot(ctx context.Context, params *CopyDBSnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { if params == nil { params = &CopyDBSnapshotInput{} diff --git a/service/rds/api_op_CreateDBCluster.go b/service/rds/api_op_CreateDBCluster.go index d216973572d..2ae283932b3 100644 --- a/service/rds/api_op_CreateDBCluster.go +++ b/service/rds/api_op_CreateDBCluster.go @@ -504,6 +504,8 @@ func newServiceMetadataMiddleware_opCreateDBCluster(region string) *awsmiddlewar } } +// PresignCreateDBCluster is used to generate a presigned HTTP Request which +// contains presigned URL, signed headers and HTTP method used. func (c *PresignClient) PresignCreateDBCluster(ctx context.Context, params *CreateDBClusterInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { if params == nil { params = &CreateDBClusterInput{} diff --git a/service/rds/api_op_CreateDBInstanceReadReplica.go b/service/rds/api_op_CreateDBInstanceReadReplica.go index 7a9a5589f25..be1147ffa59 100644 --- a/service/rds/api_op_CreateDBInstanceReadReplica.go +++ b/service/rds/api_op_CreateDBInstanceReadReplica.go @@ -514,6 +514,8 @@ func newServiceMetadataMiddleware_opCreateDBInstanceReadReplica(region string) * } } +// PresignCreateDBInstanceReadReplica is used to generate a presigned HTTP Request +// which contains presigned URL, signed headers and HTTP method used. func (c *PresignClient) PresignCreateDBInstanceReadReplica(ctx context.Context, params *CreateDBInstanceReadReplicaInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { if params == nil { params = &CreateDBInstanceReadReplicaInput{} diff --git a/service/s3/api_client.go b/service/s3/api_client.go index 3f59080fb8c..8c6d60cdf54 100644 --- a/service/s3/api_client.go +++ b/service/s3/api_client.go @@ -259,14 +259,26 @@ func addRequestResponseLogging(stack *middleware.Stack, o Options) error { }, middleware.After) } +// HTTPPresignerV4 represents presigner interface used by presign url client +type HTTPPresignerV4 interface { + PresignHTTP( + ctx context.Context, credentials aws.Credentials, r *http.Request, + payloadHash string, service string, region string, signingTime time.Time, + ) (url string, signedHeader http.Header, err error) +} + // PresignOptions represents the presign client options type PresignOptions struct { + // ClientOptions are list of functional options to mutate client options used by // presign client ClientOptions []func(*Options) + // Presigner is the presigner used by the presign url client - Presigner v4.HTTPPresigner - // Expires sets the expiration duration for the generated presign url + Presigner HTTPPresignerV4 + + // Expires sets the expiration duration for the generated presign url. This should + // be the duration in seconds the presigned URL should be considered valid for. Expires time.Duration } @@ -297,10 +309,21 @@ func (w withPresignExpires) options(o *PresignOptions) { // PresignClient represents the presign url client type PresignClient struct { client *Client - presigner v4.HTTPPresigner + presigner HTTPPresignerV4 expires time.Duration } +func newPresignClient(client *Client, options PresignOptions) *PresignClient { + if options.Presigner == nil { + options.Presigner = v4.NewSigner() + } + return &PresignClient{ + client: client, + presigner: options.Presigner, + expires: options.Expires, + } +} + // NewPresignClient generates a presign client using provided Client options and // presign options func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *PresignClient { @@ -309,16 +332,7 @@ func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *Presign fn(&presignOptions) } client := New(options, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - expires: presignOptions.Expires, - } + return newPresignClient(client, presignOptions) } // NewPresignClientWrapper generates a presign client using provided API Client and @@ -329,16 +343,7 @@ func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *Presig fn(&presignOptions) } client := copyAPIClient(c, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - expires: presignOptions.Expires, - } + return newPresignClient(client, presignOptions) } // NewPresignClientFromConfig generates a presign client using provided AWS config @@ -349,20 +354,13 @@ func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) fn(&presignOptions) } client := NewFromConfig(cfg, presignOptions.ClientOptions...) - var presigner v4.HTTPPresigner - if presignOptions.Presigner != nil { - presigner = presignOptions.Presigner - } else { - presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, presigner: presigner, - expires: presignOptions.Expires, - } + return newPresignClient(client, presignOptions) } + func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { return New(c.options, optFns...) } + func (c *PresignClient) convertToPresignMiddleware(stack *middleware.Stack, options Options) (err error) { stack.Finalize.Clear() stack.Deserialize.Clear() diff --git a/service/s3/api_op_PutObject.go b/service/s3/api_op_PutObject.go index e66bb9a73dd..2e758378e85 100644 --- a/service/s3/api_op_PutObject.go +++ b/service/s3/api_op_PutObject.go @@ -426,6 +426,8 @@ func addPutObjectUpdateEndpoint(stack *middleware.Stack, options Options) error }) } +// PresignPutObject is used to generate a presigned HTTP Request which contains +// presigned URL, signed headers and HTTP method used. func (c *PresignClient) PresignPutObject(ctx context.Context, params *PutObjectInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { if params == nil { params = &PutObjectInput{} From ec29d825c7c9ea715d9929f9ee34daf45c609c1c Mon Sep 17 00:00:00 2001 From: skotambkar Date: Mon, 16 Nov 2020 23:25:24 -0800 Subject: [PATCH 11/24] feedback changes --- .../AwsHttpPresignURLClientGenerator.java | 258 +++++++++--------- .../customization/PresignURLAutoFill.java | 136 +++++---- service/internal/presigned-url/middleware.go | 17 +- .../customizations/presigned_expires.go | 6 +- 4 files changed, 223 insertions(+), 194 deletions(-) diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java index b79babeccfe..bd0c59da458 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java @@ -19,7 +19,10 @@ import java.util.Map; import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; import software.amazon.smithy.aws.go.codegen.customization.AwsCustomGoDependency; +import software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill; import software.amazon.smithy.aws.traits.ServiceTrait; import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait; import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait; @@ -33,13 +36,24 @@ import software.amazon.smithy.go.codegen.SymbolUtils; import software.amazon.smithy.go.codegen.integration.GoIntegration; import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.StreamingTrait; import software.amazon.smithy.utils.MapUtils; import software.amazon.smithy.utils.SetUtils; +/** + * AwsHttpPresignURLClientGenerator class is a runtime plugin integration class + * that generates code for presign URL clients and associated presign operations. + *

+ * This class pulls in a static list from PresignURLAutofill customization which + * rely on the generated presigned url client and operation. This is done to + * deduplicate the listing but make this class dependent on presence of PresignURLAutofill + * class as a composition. + */ public class AwsHttpPresignURLClientGenerator implements GoIntegration { // constants private static final String CONVERT_TO_PRESIGN_MIDDLEWARE_NAME = "convertToPresignMiddleware"; @@ -47,11 +61,7 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration { private static final String PRESIGN_CLIENT = "PresignClient"; private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true); - private static final String NEW_CLIENT_INTERNAL = "newPresignClient"; private static final String NEW_CLIENT = "NewPresignClient"; - private static final String NEW_CLIENT_FROM_SERVICE = "NewPresignClientWrapper"; - private static final String NEW_CLIENT_FROM_CONFIG = "NewPresignClientFromConfig"; - private static final String PRESIGN_OPTIONS = "PresignOptions"; private static final Symbol presignOptionsSymbol = buildSymbol(PRESIGN_OPTIONS, true); @@ -64,32 +74,29 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration { private static final Symbol v4NewPresignerSymbol = SymbolUtils.createPointableSymbolBuilder( "NewSigner", AwsGoDependency.AWS_SIGNER_V4 ).build(); + private static final Symbol v4PresignedHTTPRequestSymbol = SymbolUtils.createPointableSymbolBuilder( + "PresignedHTTPRequest", AwsGoDependency.AWS_SIGNER_V4 + ).build(); - // map of service to list of operations for which presignedURL client and operation should - // be generated. - private static final Map> PRESIGNER_MAP = MapUtils.of( - ShapeId.from("com.amazonaws.s3#AmazonS3"), SetUtils.of( - ShapeId.from("com.amazonaws.s3#PutObject")), - - ShapeId.from("com.amazonaws.rds#AmazonRDSv19"), SetUtils.of( - ShapeId.from("com.amazonaws.rds#CopyDBSnapshot"), - ShapeId.from("com.amazonaws.rds#CreateDBInstanceReadReplica"), - ShapeId.from("com.amazonaws.rds#CopyDBClusterSnapshot"), - ShapeId.from("com.amazonaws.rds#CreateDBCluster")), + // constant map with service to list of operation for which presignedURL client and operation must be generated. + private static final Map> presignedClientMap = MapUtils.of( + ShapeId.from("com.amazonaws.s3#AmazonS3"), SetUtils.of(ShapeId.from("com.amazonaws.s3#PutObject")) + ); - ShapeId.from("com.amazonaws.ec2#AmazonEC2"), SetUtils.of( - ShapeId.from("com.amazonaws.ec2#CopySnapshot")) + private static final String addAsUnsignedPayloadName(String operationName) { + return String.format("add%sPayloadAsUnsigned", operationName); + } - // TODO other services - ); + // map of service to list of operations for which presignedURL client and operation should + // be generated. + public static Map> PRESIGNER_MAP = new TreeMap<>(); // build pointable symbols private static Symbol buildSymbol(String name, boolean exported) { if (!exported) { name = Character.toLowerCase(name.charAt(0)) + name.substring(1); } - return SymbolUtils.createPointableSymbolBuilder(name). - build(); + return SymbolUtils.createPointableSymbolBuilder(name).build(); } /** @@ -112,32 +119,27 @@ private static final void processFunctionalOptions( }).insertTrailingNewline(); } - /** - * variables needed in scope: - * * client - * * presignOptions - *

- * generates code to assign client, presigner and return a new presign client - * - * @param writer the writer to write to - */ - private final void returnPresignClientConstructor( - GoWriter writer, - Model model, - ServiceShape serviceShape - ) { - writer.openBlock("if presignOptions.Presigner == nil {", "}", () -> { - writer.write("presignOptions.Presigner = $T()", v4NewPresignerSymbol); - }); - - writer.openBlock("return &$L{", "}", presignClientSymbol, () -> { - writer.write("client: client,"); - writer.write("presigner: presignOptions.Presigner,"); - // if s3 assign expires value on client - if (isS3ServiceShape(model, serviceShape)) { - writer.write("expires: presignOptions.Expires,"); + @Override + public void processFinalizedModel(GoSettings settings, Model model) { + PRESIGNER_MAP.putAll(presignedClientMap); + + // update map for presign client/operation generation to include + // service/operations that use PresignURLAutoFill customization class. + Map> autofillMap = PresignURLAutoFill.SERVICE_TO_OPERATION_MAP; + for (ShapeId service : autofillMap.keySet()) { + if (!PRESIGNER_MAP.containsKey(service)) { + PRESIGNER_MAP.put(service, autofillMap.get(service)); + } else { + Set operations = new TreeSet<>(); + for (ShapeId operation : PRESIGNER_MAP.get(service)) { + operations.add(operation); + } + for (ShapeId operation : autofillMap.get(service)) { + operations.add(operation); + } + PRESIGNER_MAP.put(service, operations); } - }); + } } @Override @@ -191,6 +193,9 @@ public void writeAdditionalFiles( goDelegator.useShapeWriter(operationShape, (writer) -> { // generate presign operation function for a client operation. writePresignOperationFunction(writer, model, symbolProvider, serviceShape, operationShape); + + // generate s3 unsigned payload middleware helper + writeS3AddAsUnsignedPayloadHelper(writer, model, symbolProvider, serviceShape, operationShape); }); } } @@ -208,14 +213,14 @@ private void writePresignOperationFunction( Symbol operationInputSymbol = symbolProvider.toSymbol(operationInputShape); writer.writeDocs( - String.format("Presign%s is used to generate a presigned HTTP Request which contains presigned URL, signed headers " - + "and HTTP method used.", operationSymbol.getName()) + String.format( + "Presign%s is used to generate a presigned HTTP Request which contains presigned URL, signed headers " + + "and HTTP method used.", operationSymbol.getName()) ); writer.openBlock( "func (c *$T) Presign$T(ctx context.Context, params $P, optFns ...func($P)) " - + "(req *v4.PresignedHTTPRequest, err error) {", - "}", - presignClientSymbol, operationSymbol, operationInputSymbol, presignOptionsSymbol, + + "($P, error) {", "}", presignClientSymbol, operationSymbol, + operationInputSymbol, presignOptionsSymbol, v4PresignedHTTPRequestSymbol, () -> { Symbol nopClient = SymbolUtils.createPointableSymbolBuilder("NopClient", SmithyGoDependency.SMITHY_HTTP_TRANSPORT) @@ -227,18 +232,16 @@ private void writePresignOperationFunction( processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); // check if presigner was set for presignerOptions - writer.openBlock("if presignOptions.Presigner != nil {", "}", () -> { - writer.write( - "c = NewPresignClientWrapper(c.client, func (o $P) { o.Presigner = presignOptions.Presigner })", - presignOptionsSymbol); + writer.openBlock("if len(optFns) != 0 {", "}", () -> { + writer.write("c = $L(c.client, optFns...)", NEW_CLIENT); }); + writer.write(""); - writer.write("clientOptFns := presignOptions.ClientOptions"); - + writer.write("clientOptFns := make([]func(o *Options), 0)"); writer.openBlock("clientOptFns = append(clientOptFns, func(o *Options) {", "})", () -> { writer.write("o.HTTPClient = &$T{}", nopClient); }); - writer.insertTrailingNewline(); + writer.write(""); Symbol withIsPresigning = SymbolUtils.createValueSymbolBuilder("WithIsPresigning", AwsCustomGoDependency.PRESIGNEDURL_CUSTOMIZATION).build(); @@ -249,13 +252,55 @@ private void writePresignOperationFunction( writer.write("$L,", OperationGenerator .getAddOperationMiddlewareFuncName(operationSymbol)); writer.write("c.$L,", CONVERT_TO_PRESIGN_MIDDLEWARE_NAME); + + // s3 should add a middleware where it switches to using unisgned payload if + // input is a stream. + if (isS3ServiceShape(model, serviceShape)) { + if (operationInputShape.members().stream().anyMatch(memberShape -> { + return memberShape.getMemberTrait(model, StreamingTrait.class).isPresent(); + })) { + writer.write("$L,", addAsUnsignedPayloadName(operationSymbol.getName())); + } + } }); - writer.write("if err != nil { return req, err }"); - writer.insertTrailingNewline(); + writer.write("if err != nil { return nil, err }"); + writer.write(""); - writer.write("out := result.(*v4.PresignedHTTPRequest)"); + writer.write("out := result.($P)", v4PresignedHTTPRequestSymbol); writer.write("return out, nil"); }); + writer.write(""); + } + + private void writeS3AddAsUnsignedPayloadHelper( + GoWriter writer, + Model model, + SymbolProvider symbolProvider, + ServiceShape serviceShape, + OperationShape operationShape + ) { + // if service is not s3, return + if (!isS3ServiceShape(model, serviceShape)) { + return; + } + + Symbol operationSymbol = symbolProvider.toSymbol(operationShape); + + Shape operationInputShape = model.expectShape(operationShape.getInput().get()); + + // return if not streaming + if (operationInputShape.members().stream().noneMatch(memberShape -> { + return memberShape.getMemberTrait(model, StreamingTrait.class).isPresent(); + })) { return; } + + writer.openBlock("func $L(stack $P, options Options) error {", "}", + addAsUnsignedPayloadName(operationSymbol.getName()), + SymbolUtils.createPointableSymbolBuilder("Stack", SmithyGoDependency.SMITHY_MIDDLEWARE).build(), + () -> { + writer.write("return $T(stack)", SymbolUtils.createValueSymbolBuilder( + "AddUnsignedPayloadMiddleware", AwsGoDependency.AWS_SIGNER_V4).build()); + }); + writer.write(""); } /** @@ -315,17 +360,16 @@ private void writeConvertToPresignMiddleware( writer.write("if err != nil { return err }"); } - // s3 service needs expires + // s3 service needs expires and sets unsignedPayload if input is stream if (isS3ServiceShape(model, serviceShape)) { Symbol expiresAsHeaderMiddleware = SymbolUtils.createValueSymbolBuilder( "AddExpiresOnPresignedURL", AwsCustomGoDependency.S3_CUSTOMIZATION).build(); - writer.openBlock("if c.expires != 0 {", "}", () -> { - writer.writeDocs("add middleware to set expiration for s3 presigned url"); - writer.write("err = stack.Build.Add(&$T{ Expires: c.expires, }, middleware.After)", - expiresAsHeaderMiddleware); - writer.write("if err != nil { return err }"); - }); + writer.writeDocs("add middleware to set expiration for s3 presigned url, " + + " if expiration is set to 0, this middleware sets a default expiration of 900 seconds"); + writer.write("err = stack.Build.Add(&$T{ Expires: c.expires, }, middleware.After)", + expiresAsHeaderMiddleware); + writer.write("if err != nil { return err }"); } writer.write("return nil"); @@ -357,70 +401,31 @@ private void writePresignClientType( writer.write("expires time.Duration"); } }); - - // generate internal constructors - writer.openBlock("func $L(client *Client, options $T) $P {", "}", - NEW_CLIENT_INTERNAL, presignOptionsSymbol, presignClientSymbol, () -> { - writer.openBlock("if options.Presigner == nil {", "}", () -> { - writer.write("options.Presigner = $T()", v4NewPresignerSymbol); - }); - - writer.openBlock("return &$L{", "}", presignClientSymbol, () -> { - writer.write("client: client,"); - writer.write("presigner: options.Presigner,"); - // if s3 assign expires value on client - if (isS3ServiceShape(model, serviceShape)) { - writer.write("expires: options.Expires,"); - } - }); - }); writer.write(""); // generate NewPresignClient - writer.writeDocs( - String.format("%s generates a presign client using provided Client options and presign options", - NEW_CLIENT) - ); - writer.openBlock("func $L(options Options, optFns ...func($P)) $P {", "}", - NEW_CLIENT, presignOptionsSymbol, presignClientSymbol, () -> { - processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); - writer.insertTrailingNewline(); - writer.write("client := New(options, presignOptions.ClientOptions...)").insertTrailingNewline(); - writer.insertTrailingNewline(); - - writer.write("return $L(client, presignOptions)", NEW_CLIENT_INTERNAL); - }).insertTrailingNewline(); - - // generate NewPresignClientWrapper writer.writeDocs( String.format("%s generates a presign client using provided API Client and presign options", - NEW_CLIENT_FROM_SERVICE) + NEW_CLIENT) ); writer.openBlock("func $L(c *Client, optFns ...func($P)) $P {", "}", - NEW_CLIENT_FROM_SERVICE, presignOptionsSymbol, presignClientSymbol, () -> { + NEW_CLIENT, presignOptionsSymbol, presignClientSymbol, () -> { processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); - writer.insertTrailingNewline(); writer.write("client := copyAPIClient(c, presignOptions.ClientOptions...)"); - writer.insertTrailingNewline(); - - writer.write("return $L(client, presignOptions)", NEW_CLIENT_INTERNAL); - }).insertTrailingNewline(); - - // generate NewPresignClientFromConfig - writer.writeDocs( - String.format("%s generates a presign client using provided AWS config and presign options", - NEW_CLIENT_FROM_CONFIG) - ); - writer.openBlock("func $L(cfg aws.Config, optFns ...func($P)) $P {", "}", - NEW_CLIENT_FROM_CONFIG, presignOptionsSymbol, presignClientSymbol, () -> { - processFunctionalOptions(writer, "optFns", "presignOptions", presignOptionsSymbol); - writer.insertTrailingNewline(); - writer.write("client := NewFromConfig(cfg, presignOptions.ClientOptions...)"); - writer.insertTrailingNewline(); - - writer.write("return $L(client, presignOptions)", NEW_CLIENT_INTERNAL); - }).insertTrailingNewline(); + writer.openBlock("if presignOptions.Presigner == nil {", "}", () -> { + writer.write("presignOptions.Presigner = $T()", v4NewPresignerSymbol); + }); + writer.write(""); + writer.openBlock("return &$L{", "}", presignClientSymbol, () -> { + writer.write("client: client,"); + writer.write("presigner: presignOptions.Presigner,"); + // if s3 assign expires value on client + if (isS3ServiceShape(model, serviceShape)) { + writer.write("expires: presignOptions.Expires,"); + } + }); + }); writer.write(""); } @@ -455,7 +460,8 @@ public void writePresignInterface( ServiceShape serviceShape ) { writer.writeDocs( - String.format("%s represents presigner interface used by presign url client", presignerInterfaceSymbol.getName()) + String.format("%s represents presigner interface used by presign url client", + presignerInterfaceSymbol.getName()) ); writer.openBlock("type $T interface {", "}", presignerInterfaceSymbol, () -> { writer.write("PresignHTTP("); @@ -464,7 +470,7 @@ public void writePresignInterface( writer.write(") (url string, signedHeader http.Header, err error)"); }); - writer.insertTrailingNewline(); + writer.write(""); } /** @@ -499,7 +505,8 @@ public void writePresignOptionType( writer.write(""); writer.writeDocs( String.format("Expires sets the expiration duration for the generated presign url. This should " - + "be the duration in seconds the presigned URL should be considered valid for." + + "be the duration in seconds the presigned URL should be considered valid for. If " + + "not set or set to zero, presign url would default to expire after 900 seconds." ) ); writer.write("Expires time.Duration"); @@ -554,4 +561,3 @@ private final boolean isS3ServiceShape(Model model, ServiceShape service) { } } -// TODO: generate tests for presigned urls diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java index b1a109b69f7..6a606746556 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/PresignURLAutoFill.java @@ -52,9 +52,17 @@ import software.amazon.smithy.utils.MapUtils; import software.amazon.smithy.utils.SetUtils; +/** + * PresignURLAutoFill represents a runtime plugin integration class + * used generate customization to autofill a presign url as + * an unexported serialized input member + */ public class PresignURLAutoFill implements GoIntegration { - private static final Logger LOGGER = Logger.getLogger(PresignURLAutoFill.class.getName()); - private static final Map> SERVICE_TO_OPERATION_MAP = MapUtils.of( + /** + * Map of service shape to Set of operation shapes that need to have this + * presigned url auto fill customization. + */ + public static final Map> SERVICE_TO_OPERATION_MAP = MapUtils.of( ShapeId.from("com.amazonaws.rds#AmazonRDSv19"), SetUtils.of( ShapeId.from("com.amazonaws.rds#CopyDBSnapshot"), ShapeId.from("com.amazonaws.rds#CreateDBInstanceReadReplica"), @@ -66,6 +74,7 @@ public class PresignURLAutoFill implements GoIntegration { // TODO other services ); + private static final Logger LOGGER = Logger.getLogger(PresignURLAutoFill.class.getName()); private final List runtimeClientPlugins = new ArrayList<>(); private static void writeMemberSetter( @@ -118,51 +127,6 @@ private static void writeMemberGetter( }); } - private static void writePresignOperationAccessor( - GoWriter writer, - SymbolProvider symbolprovider, - OperationShape operation, - StructureShape input - ) { - Symbol operationSymbol = symbolprovider.toSymbol(operation); - Symbol inputSymbol = symbolprovider.toSymbol(input); - - Symbol removeMiddleware = SymbolUtils.createValueSymbolBuilder("RemoveMiddleware", - AwsCustomGoDependency.PRESIGNEDURL_CUSTOMIZATION).build(); - - writer.openBlock( - "func $L(ctx context.Context, client interface{}, region string, params interface{}) " - + "(req *v4.PresignedHTTPRequest, err error) {", "}", - presignFuncName(operationSymbol.getName(), false), - () -> { - writer.addUseImports(SmithyGoDependency.FMT); - // check input - writer.write("input, ok := params.($P)", inputSymbol); - writer.openBlock("if !ok {", "}", () -> { - writer.write("return req, fmt.Errorf(\"expect $P type, got %T\", params)", inputSymbol); - }); - - // check client - writer.write("c, ok := client.(*PresignClient)"); - writer.openBlock("if !ok {", "}", () -> { - writer.write("return req, fmt.Errorf(\"expect *PresignClient type, got %T\", client)"); - }); - - // generate client options - writer.openBlock("optFn := func(o *Options) {", "}", () -> { - writer.write("o.Region = region"); - writer.write("o.APIOptions = append(o.APIOptions, $T)", removeMiddleware); - }); - - // getPresignAPIOptions - writer.write("presignOptFn := WithPresignClientFromClientOptions(optFn)"); - - // call the exported function - writer.write("return c.$L(ctx, input, presignOptFn)", - presignFuncName(operationSymbol.getName(), true)); - }); - } - private static String addPresignMiddlewareFuncName(String operationName) { return String.format("add%sPresignURLMiddleware", operationName); } @@ -179,8 +143,12 @@ private static String copyInputFuncName(String inputName) { return String.format("copy%sForPresign", inputName); } - private static String presignFuncName(String operationName, boolean exported) { - return exported ? String.format("Presign%s", operationName) : String.format("presign%s", operationName); + private static String presignFuncName(String operationName) { + return String.format("Presign%s", operationName); + } + + private static String autofillPresignClient(String operationName) { + return String.format("presignAutoFill%sClient", operationName); } /** @@ -272,10 +240,12 @@ public void writeAdditionalFiles( // Members used by the customization need abstract getter and setters writeMemberAccessor(writer, symbolProvider, operation, input); - writePresignOperationAccessor(writer, symbolProvider, operation, input); // Generate the presign client writePresignClientCustomization(writer, settings, model, symbolProvider, operation, input); + + // Generate the autofill presign client and its PresignURL method + writeAutofillPresignClient(writer, symbolProvider, operation, input); }); goDelegator.useShapeTestWriter(operation, (writer) -> { @@ -395,22 +365,69 @@ private void writePresignClientCustomization( () -> { writer.openBlock("return $T(stack, $T{", "})", addMiddleware, addMiddlewareOptions, () -> { writer.openBlock("Accessor: $T{", "},", parameterAccessor, () -> { - writer.write("GetPresignedURL: $L,", + writer.write("GetPresignedURL: $L, \n", getterFuncName(operationSymbol.getName(), presignURLMember)); - writer.write("GetSourceRegion: $L,", + writer.write("GetSourceRegion: $L, \n", getterFuncName(operationSymbol.getName(), srcRegionMember)); - writer.write("CopyInput: $L,", copyInputFuncName(inputSymbol.getName())); - writer.write("SetDestinationRegion: $L,", + writer.write("CopyInput: $L, \n", copyInputFuncName(inputSymbol.getName())); + writer.write("SetDestinationRegion: $L,\n", setterFuncName(operationSymbol.getName(), dstRegionMember)); - writer.write("SetPresignedURL: $L,", + writer.write("SetPresignedURL: $L, \n", setterFuncName(operationSymbol.getName(), presignURLMember)); - writer.write("PresignOperation: $L,", - presignFuncName(operationSymbol.getName(), false)); }); - // Replace with type wrapping presigner for generic signature - writer.write("PresignClient: NewPresignClient(options),"); + writer.write("Presigner: &$L{ client: NewPresignClient(New(options))}, \n", + autofillPresignClient(operationSymbol.getName())); + }); + }); + } + + + private void writeAutofillPresignClient( + GoWriter writer, + SymbolProvider symbolprovider, + OperationShape operation, + StructureShape input + ) { + Symbol operationSymbol = symbolprovider.toSymbol(operation); + Symbol inputSymbol = symbolprovider.toSymbol(input); + Symbol removeMiddleware = SymbolUtils.createValueSymbolBuilder("RemoveMiddleware", + AwsCustomGoDependency.PRESIGNEDURL_CUSTOMIZATION).build(); + + // generate autofill presign client + writer.openBlock("type $L struct {", "}", + autofillPresignClient(operationSymbol.getName()), () -> { + writer.write("client *PresignClient"); + }); + + writer.write(""); + + // generate PresignURL method that satisfies URLPresigner interface of middleware + writer.writeDocs("PresignURL is a middleware accessor that satisfies URLPresigner interface."); + writer.openBlock("func (c *$L) PresignURL(ctx context.Context, srcRegion string, params interface{}) " + + "(*v4.PresignedHTTPRequest, error) {", "}", + + autofillPresignClient(operationSymbol.getName()), () -> { + writer.addUseImports(SmithyGoDependency.FMT); + // check input + writer.write("input, ok := params.($P)", inputSymbol); + writer.openBlock("if !ok {", "}", () -> { + writer.write("return nil, fmt.Errorf(\"expect $P type, got %T\", params)", inputSymbol); }); + + // generate client options + writer.openBlock("optFn := func(o *Options) {", "}", () -> { + writer.write("o.Region = srcRegion"); + writer.write("o.APIOptions = append(o.APIOptions, $T)", removeMiddleware); + }); + + // getPresignAPIOptions + writer.write("presignOptFn := WithPresignClientFromClientOptions(optFn)"); + + // call the exported function + writer.write("return c.client.$L(ctx, input, presignOptFn)", + presignFuncName(operationSymbol.getName())); }); + writer.write(""); } private void writePresignClientCustomizationTest( @@ -450,4 +467,5 @@ private void writePresignClientCustomizationTest( writer.write(template); } + } diff --git a/service/internal/presigned-url/middleware.go b/service/internal/presigned-url/middleware.go index 9e732f45e88..f68be9ef643 100644 --- a/service/internal/presigned-url/middleware.go +++ b/service/internal/presigned-url/middleware.go @@ -10,6 +10,13 @@ import ( "github.com/awslabs/smithy-go/middleware" ) +// URLPresigner provides the interface to presign the input parameters in to a +// presigned URL. +type URLPresigner interface { + // PresignURL presigns a URL. + PresignURL(ctx context.Context, srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error) +} + // ParameterAccessor provides an collection of accessor to for retrieving and // setting the values needed to PresignedURL generation type ParameterAccessor struct { @@ -27,10 +34,6 @@ type ParameterAccessor struct { // SetPresignedURL accessor points to a function that sets presigned url on api input struct SetPresignedURL func(interface{}, string) error - - // PresignOperation is the presign function accessor - PresignOperation func(ctx context.Context, client interface{}, - srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error) } // Options provides the set of options needed by the presigned URL middleware. @@ -38,8 +41,8 @@ type Options struct { // Accessor are the parameter accessors used by this middleware Accessor ParameterAccessor - // PresignClient is the presigner client used to presign an operation request. - PresignClient interface{} + // Presigner is the URLPresigner used by the middleware + Presigner URLPresigner } // AddMiddleware adds the Presign URL middleware to the middleware stack. @@ -93,7 +96,7 @@ func (m *presign) HandleInitialize( return out, metadata, fmt.Errorf("presign middleware failed, %w", err) } - presignedReq, err := m.options.Accessor.PresignOperation(ctx, m.options.PresignClient, srcRegion, paramCpy) + presignedReq, err := m.options.Presigner.PresignURL(ctx, srcRegion, paramCpy) if err != nil { return out, metadata, fmt.Errorf("unable to create presigned URL, %w", err) } diff --git a/service/s3/internal/customizations/presigned_expires.go b/service/s3/internal/customizations/presigned_expires.go index 54121c883a1..2ea6318d4a9 100644 --- a/service/s3/internal/customizations/presigned_expires.go +++ b/service/s3/internal/customizations/presigned_expires.go @@ -11,11 +11,12 @@ import ( ) // AddExpiresOnPresignedURL represents a build middleware used to assign -// expiration on a presigned URL +// expiration on a presigned URL. type AddExpiresOnPresignedURL struct { // Expires is time.Duration within which presigned url should be expired. // This should be the duration in seconds the presigned URL should be considered valid for. + // By default the S3 presigned url expires in 15 minutes ie. 900 seconds. Expires time.Duration } @@ -30,7 +31,8 @@ func (m *AddExpiresOnPresignedURL) HandleBuild(ctx context.Context, in middlewar ) { // if expiration is unset skip this middleware if m.Expires == 0 { - return next.HandleBuild(ctx, in) + // default to 15 * time.Minutes + m.Expires = 15 * time.Minute } req, ok := in.Request.(*smithyhttp.Request) From e55671e232b1bc9445b104b4e496efb8005f2c9e Mon Sep 17 00:00:00 2001 From: skotambkar Date: Mon, 16 Nov 2020 23:26:32 -0800 Subject: [PATCH 12/24] regenerated client for s3, ec2, rds --- service/ec2/api_client.go | 41 +++---------- service/ec2/api_op_CopySnapshot.go | 60 +++++++++++-------- service/rds/api_client.go | 41 +++---------- service/rds/api_op_CopyDBClusterSnapshot.go | 60 +++++++++++-------- service/rds/api_op_CopyDBSnapshot.go | 60 +++++++++++-------- service/rds/api_op_CreateDBCluster.go | 60 +++++++++++-------- .../rds/api_op_CreateDBInstanceReadReplica.go | 60 +++++++++++-------- service/rds/go.sum | 8 +++ service/s3/api_client.go | 57 +++++------------- service/s3/api_op_PutObject.go | 18 ++++-- service/s3/go.sum | 13 ++++ 11 files changed, 236 insertions(+), 242 deletions(-) diff --git a/service/ec2/api_client.go b/service/ec2/api_client.go index a9b71c53049..3b4c1d0b7be 100644 --- a/service/ec2/api_client.go +++ b/service/ec2/api_client.go @@ -285,47 +285,22 @@ type PresignClient struct { presigner HTTPPresignerV4 } -func newPresignClient(client *Client, options PresignOptions) *PresignClient { - if options.Presigner == nil { - options.Presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, - presigner: options.Presigner, - } -} - -// NewPresignClient generates a presign client using provided Client options and -// presign options -func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *PresignClient { - var presignOptions PresignOptions - for _, fn := range optFns { - fn(&presignOptions) - } - client := New(options, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) -} - -// NewPresignClientWrapper generates a presign client using provided API Client and +// NewPresignClient generates a presign client using provided API Client and // presign options -func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *PresignClient { +func NewPresignClient(c *Client, optFns ...func(*PresignOptions)) *PresignClient { var presignOptions PresignOptions for _, fn := range optFns { fn(&presignOptions) } client := copyAPIClient(c, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) -} + if presignOptions.Presigner == nil { + presignOptions.Presigner = v4.NewSigner() + } -// NewPresignClientFromConfig generates a presign client using provided AWS config -// and presign options -func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) *PresignClient { - var presignOptions PresignOptions - for _, fn := range optFns { - fn(&presignOptions) + return &PresignClient{ + client: client, + presigner: presignOptions.Presigner, } - client := NewFromConfig(cfg, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) } func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { diff --git a/service/ec2/api_op_CopySnapshot.go b/service/ec2/api_op_CopySnapshot.go index d16d7d7ec79..dec72096519 100644 --- a/service/ec2/api_op_CopySnapshot.go +++ b/service/ec2/api_op_CopySnapshot.go @@ -238,34 +238,39 @@ func setCopySnapshotdestinationRegion(params interface{}, value string) error { input.destinationRegion = &value return nil } -func presignCopySnapshot(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { +func addCopySnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { + return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ + Accessor: presignedurlcust.ParameterAccessor{ + GetPresignedURL: getCopySnapshotPresignedUrl, + + GetSourceRegion: getCopySnapshotSourceRegion, + + CopyInput: copyCopySnapshotInputForPresign, + + SetDestinationRegion: setCopySnapshotdestinationRegion, + + SetPresignedURL: setCopySnapshotPresignedUrl, + }, + Presigner: &presignAutoFillCopySnapshotClient{client: NewPresignClient(New(options))}, + }) +} + +type presignAutoFillCopySnapshotClient struct { + client *PresignClient +} + +// PresignURL is a middleware accessor that satisfies URLPresigner interface. +func (c *presignAutoFillCopySnapshotClient) PresignURL(ctx context.Context, srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error) { input, ok := params.(*CopySnapshotInput) if !ok { - return req, fmt.Errorf("expect *CopySnapshotInput type, got %T", params) - } - c, ok := client.(*PresignClient) - if !ok { - return req, fmt.Errorf("expect *PresignClient type, got %T", client) + return nil, fmt.Errorf("expect *CopySnapshotInput type, got %T", params) } optFn := func(o *Options) { - o.Region = region + o.Region = srcRegion o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } presignOptFn := WithPresignClientFromClientOptions(optFn) - return c.PresignCopySnapshot(ctx, input, presignOptFn) -} -func addCopySnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { - return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ - Accessor: presignedurlcust.ParameterAccessor{ - GetPresignedURL: getCopySnapshotPresignedUrl, - GetSourceRegion: getCopySnapshotSourceRegion, - CopyInput: copyCopySnapshotInputForPresign, - SetDestinationRegion: setCopySnapshotdestinationRegion, - SetPresignedURL: setCopySnapshotPresignedUrl, - PresignOperation: presignCopySnapshot, - }, - PresignClient: NewPresignClient(options), - }) + return c.client.PresignCopySnapshot(ctx, input, presignOptFn) } func newServiceMetadataMiddleware_opCopySnapshot(region string) *awsmiddleware.RegisterServiceMetadata { @@ -279,7 +284,7 @@ func newServiceMetadataMiddleware_opCopySnapshot(region string) *awsmiddleware.R // PresignCopySnapshot is used to generate a presigned HTTP Request which contains // presigned URL, signed headers and HTTP method used. -func (c *PresignClient) PresignCopySnapshot(ctx context.Context, params *CopySnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { +func (c *PresignClient) PresignCopySnapshot(ctx context.Context, params *CopySnapshotInput, optFns ...func(*PresignOptions)) (*v4.PresignedHTTPRequest, error) { if params == nil { params = &CopySnapshotInput{} } @@ -287,21 +292,24 @@ func (c *PresignClient) PresignCopySnapshot(ctx context.Context, params *CopySna for _, fn := range optFns { fn(&presignOptions) } - if presignOptions.Presigner != nil { - c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + if len(optFns) != 0 { + c = NewPresignClient(c.client, optFns...) } - clientOptFns := presignOptions.ClientOptions + + clientOptFns := make([]func(o *Options), 0) clientOptFns = append(clientOptFns, func(o *Options) { o.HTTPClient = &smithyhttp.NopClient{} }) + ctx = presignedurlcust.WithIsPresigning(ctx) result, _, err := c.client.invokeOperation(ctx, "CopySnapshot", params, clientOptFns, addOperationCopySnapshotMiddlewares, c.convertToPresignMiddleware, ) if err != nil { - return req, err + return nil, err } + out := result.(*v4.PresignedHTTPRequest) return out, nil } diff --git a/service/rds/api_client.go b/service/rds/api_client.go index b00e75eb627..cd36a219da5 100644 --- a/service/rds/api_client.go +++ b/service/rds/api_client.go @@ -265,47 +265,22 @@ type PresignClient struct { presigner HTTPPresignerV4 } -func newPresignClient(client *Client, options PresignOptions) *PresignClient { - if options.Presigner == nil { - options.Presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, - presigner: options.Presigner, - } -} - -// NewPresignClient generates a presign client using provided Client options and -// presign options -func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *PresignClient { - var presignOptions PresignOptions - for _, fn := range optFns { - fn(&presignOptions) - } - client := New(options, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) -} - -// NewPresignClientWrapper generates a presign client using provided API Client and +// NewPresignClient generates a presign client using provided API Client and // presign options -func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *PresignClient { +func NewPresignClient(c *Client, optFns ...func(*PresignOptions)) *PresignClient { var presignOptions PresignOptions for _, fn := range optFns { fn(&presignOptions) } client := copyAPIClient(c, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) -} + if presignOptions.Presigner == nil { + presignOptions.Presigner = v4.NewSigner() + } -// NewPresignClientFromConfig generates a presign client using provided AWS config -// and presign options -func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) *PresignClient { - var presignOptions PresignOptions - for _, fn := range optFns { - fn(&presignOptions) + return &PresignClient{ + client: client, + presigner: presignOptions.Presigner, } - client := NewFromConfig(cfg, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) } func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { diff --git a/service/rds/api_op_CopyDBClusterSnapshot.go b/service/rds/api_op_CopyDBClusterSnapshot.go index 0a03b67aad3..8662a7a11a8 100644 --- a/service/rds/api_op_CopyDBClusterSnapshot.go +++ b/service/rds/api_op_CopyDBClusterSnapshot.go @@ -328,34 +328,39 @@ func setCopyDBClusterSnapshotdestinationRegion(params interface{}, value string) input.destinationRegion = &value return nil } -func presignCopyDBClusterSnapshot(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { +func addCopyDBClusterSnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { + return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ + Accessor: presignedurlcust.ParameterAccessor{ + GetPresignedURL: getCopyDBClusterSnapshotPreSignedUrl, + + GetSourceRegion: getCopyDBClusterSnapshotSourceRegion, + + CopyInput: copyCopyDBClusterSnapshotInputForPresign, + + SetDestinationRegion: setCopyDBClusterSnapshotdestinationRegion, + + SetPresignedURL: setCopyDBClusterSnapshotPreSignedUrl, + }, + Presigner: &presignAutoFillCopyDBClusterSnapshotClient{client: NewPresignClient(New(options))}, + }) +} + +type presignAutoFillCopyDBClusterSnapshotClient struct { + client *PresignClient +} + +// PresignURL is a middleware accessor that satisfies URLPresigner interface. +func (c *presignAutoFillCopyDBClusterSnapshotClient) PresignURL(ctx context.Context, srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error) { input, ok := params.(*CopyDBClusterSnapshotInput) if !ok { - return req, fmt.Errorf("expect *CopyDBClusterSnapshotInput type, got %T", params) - } - c, ok := client.(*PresignClient) - if !ok { - return req, fmt.Errorf("expect *PresignClient type, got %T", client) + return nil, fmt.Errorf("expect *CopyDBClusterSnapshotInput type, got %T", params) } optFn := func(o *Options) { - o.Region = region + o.Region = srcRegion o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } presignOptFn := WithPresignClientFromClientOptions(optFn) - return c.PresignCopyDBClusterSnapshot(ctx, input, presignOptFn) -} -func addCopyDBClusterSnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { - return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ - Accessor: presignedurlcust.ParameterAccessor{ - GetPresignedURL: getCopyDBClusterSnapshotPreSignedUrl, - GetSourceRegion: getCopyDBClusterSnapshotSourceRegion, - CopyInput: copyCopyDBClusterSnapshotInputForPresign, - SetDestinationRegion: setCopyDBClusterSnapshotdestinationRegion, - SetPresignedURL: setCopyDBClusterSnapshotPreSignedUrl, - PresignOperation: presignCopyDBClusterSnapshot, - }, - PresignClient: NewPresignClient(options), - }) + return c.client.PresignCopyDBClusterSnapshot(ctx, input, presignOptFn) } func newServiceMetadataMiddleware_opCopyDBClusterSnapshot(region string) *awsmiddleware.RegisterServiceMetadata { @@ -369,7 +374,7 @@ func newServiceMetadataMiddleware_opCopyDBClusterSnapshot(region string) *awsmid // PresignCopyDBClusterSnapshot is used to generate a presigned HTTP Request which // contains presigned URL, signed headers and HTTP method used. -func (c *PresignClient) PresignCopyDBClusterSnapshot(ctx context.Context, params *CopyDBClusterSnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { +func (c *PresignClient) PresignCopyDBClusterSnapshot(ctx context.Context, params *CopyDBClusterSnapshotInput, optFns ...func(*PresignOptions)) (*v4.PresignedHTTPRequest, error) { if params == nil { params = &CopyDBClusterSnapshotInput{} } @@ -377,21 +382,24 @@ func (c *PresignClient) PresignCopyDBClusterSnapshot(ctx context.Context, params for _, fn := range optFns { fn(&presignOptions) } - if presignOptions.Presigner != nil { - c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + if len(optFns) != 0 { + c = NewPresignClient(c.client, optFns...) } - clientOptFns := presignOptions.ClientOptions + + clientOptFns := make([]func(o *Options), 0) clientOptFns = append(clientOptFns, func(o *Options) { o.HTTPClient = &smithyhttp.NopClient{} }) + ctx = presignedurlcust.WithIsPresigning(ctx) result, _, err := c.client.invokeOperation(ctx, "CopyDBClusterSnapshot", params, clientOptFns, addOperationCopyDBClusterSnapshotMiddlewares, c.convertToPresignMiddleware, ) if err != nil { - return req, err + return nil, err } + out := result.(*v4.PresignedHTTPRequest) return out, nil } diff --git a/service/rds/api_op_CopyDBSnapshot.go b/service/rds/api_op_CopyDBSnapshot.go index 4e037541c07..a839500bc76 100644 --- a/service/rds/api_op_CopyDBSnapshot.go +++ b/service/rds/api_op_CopyDBSnapshot.go @@ -281,34 +281,39 @@ func setCopyDBSnapshotdestinationRegion(params interface{}, value string) error input.destinationRegion = &value return nil } -func presignCopyDBSnapshot(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { +func addCopyDBSnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { + return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ + Accessor: presignedurlcust.ParameterAccessor{ + GetPresignedURL: getCopyDBSnapshotPreSignedUrl, + + GetSourceRegion: getCopyDBSnapshotSourceRegion, + + CopyInput: copyCopyDBSnapshotInputForPresign, + + SetDestinationRegion: setCopyDBSnapshotdestinationRegion, + + SetPresignedURL: setCopyDBSnapshotPreSignedUrl, + }, + Presigner: &presignAutoFillCopyDBSnapshotClient{client: NewPresignClient(New(options))}, + }) +} + +type presignAutoFillCopyDBSnapshotClient struct { + client *PresignClient +} + +// PresignURL is a middleware accessor that satisfies URLPresigner interface. +func (c *presignAutoFillCopyDBSnapshotClient) PresignURL(ctx context.Context, srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error) { input, ok := params.(*CopyDBSnapshotInput) if !ok { - return req, fmt.Errorf("expect *CopyDBSnapshotInput type, got %T", params) - } - c, ok := client.(*PresignClient) - if !ok { - return req, fmt.Errorf("expect *PresignClient type, got %T", client) + return nil, fmt.Errorf("expect *CopyDBSnapshotInput type, got %T", params) } optFn := func(o *Options) { - o.Region = region + o.Region = srcRegion o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } presignOptFn := WithPresignClientFromClientOptions(optFn) - return c.PresignCopyDBSnapshot(ctx, input, presignOptFn) -} -func addCopyDBSnapshotPresignURLMiddleware(stack *middleware.Stack, options Options) error { - return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ - Accessor: presignedurlcust.ParameterAccessor{ - GetPresignedURL: getCopyDBSnapshotPreSignedUrl, - GetSourceRegion: getCopyDBSnapshotSourceRegion, - CopyInput: copyCopyDBSnapshotInputForPresign, - SetDestinationRegion: setCopyDBSnapshotdestinationRegion, - SetPresignedURL: setCopyDBSnapshotPreSignedUrl, - PresignOperation: presignCopyDBSnapshot, - }, - PresignClient: NewPresignClient(options), - }) + return c.client.PresignCopyDBSnapshot(ctx, input, presignOptFn) } func newServiceMetadataMiddleware_opCopyDBSnapshot(region string) *awsmiddleware.RegisterServiceMetadata { @@ -322,7 +327,7 @@ func newServiceMetadataMiddleware_opCopyDBSnapshot(region string) *awsmiddleware // PresignCopyDBSnapshot is used to generate a presigned HTTP Request which // contains presigned URL, signed headers and HTTP method used. -func (c *PresignClient) PresignCopyDBSnapshot(ctx context.Context, params *CopyDBSnapshotInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { +func (c *PresignClient) PresignCopyDBSnapshot(ctx context.Context, params *CopyDBSnapshotInput, optFns ...func(*PresignOptions)) (*v4.PresignedHTTPRequest, error) { if params == nil { params = &CopyDBSnapshotInput{} } @@ -330,21 +335,24 @@ func (c *PresignClient) PresignCopyDBSnapshot(ctx context.Context, params *CopyD for _, fn := range optFns { fn(&presignOptions) } - if presignOptions.Presigner != nil { - c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + if len(optFns) != 0 { + c = NewPresignClient(c.client, optFns...) } - clientOptFns := presignOptions.ClientOptions + + clientOptFns := make([]func(o *Options), 0) clientOptFns = append(clientOptFns, func(o *Options) { o.HTTPClient = &smithyhttp.NopClient{} }) + ctx = presignedurlcust.WithIsPresigning(ctx) result, _, err := c.client.invokeOperation(ctx, "CopyDBSnapshot", params, clientOptFns, addOperationCopyDBSnapshotMiddlewares, c.convertToPresignMiddleware, ) if err != nil { - return req, err + return nil, err } + out := result.(*v4.PresignedHTTPRequest) return out, nil } diff --git a/service/rds/api_op_CreateDBCluster.go b/service/rds/api_op_CreateDBCluster.go index 2ae283932b3..78e89d9ed0f 100644 --- a/service/rds/api_op_CreateDBCluster.go +++ b/service/rds/api_op_CreateDBCluster.go @@ -465,34 +465,39 @@ func setCreateDBClusterdestinationRegion(params interface{}, value string) error input.destinationRegion = &value return nil } -func presignCreateDBCluster(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { +func addCreateDBClusterPresignURLMiddleware(stack *middleware.Stack, options Options) error { + return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ + Accessor: presignedurlcust.ParameterAccessor{ + GetPresignedURL: getCreateDBClusterPreSignedUrl, + + GetSourceRegion: getCreateDBClusterSourceRegion, + + CopyInput: copyCreateDBClusterInputForPresign, + + SetDestinationRegion: setCreateDBClusterdestinationRegion, + + SetPresignedURL: setCreateDBClusterPreSignedUrl, + }, + Presigner: &presignAutoFillCreateDBClusterClient{client: NewPresignClient(New(options))}, + }) +} + +type presignAutoFillCreateDBClusterClient struct { + client *PresignClient +} + +// PresignURL is a middleware accessor that satisfies URLPresigner interface. +func (c *presignAutoFillCreateDBClusterClient) PresignURL(ctx context.Context, srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error) { input, ok := params.(*CreateDBClusterInput) if !ok { - return req, fmt.Errorf("expect *CreateDBClusterInput type, got %T", params) - } - c, ok := client.(*PresignClient) - if !ok { - return req, fmt.Errorf("expect *PresignClient type, got %T", client) + return nil, fmt.Errorf("expect *CreateDBClusterInput type, got %T", params) } optFn := func(o *Options) { - o.Region = region + o.Region = srcRegion o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } presignOptFn := WithPresignClientFromClientOptions(optFn) - return c.PresignCreateDBCluster(ctx, input, presignOptFn) -} -func addCreateDBClusterPresignURLMiddleware(stack *middleware.Stack, options Options) error { - return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ - Accessor: presignedurlcust.ParameterAccessor{ - GetPresignedURL: getCreateDBClusterPreSignedUrl, - GetSourceRegion: getCreateDBClusterSourceRegion, - CopyInput: copyCreateDBClusterInputForPresign, - SetDestinationRegion: setCreateDBClusterdestinationRegion, - SetPresignedURL: setCreateDBClusterPreSignedUrl, - PresignOperation: presignCreateDBCluster, - }, - PresignClient: NewPresignClient(options), - }) + return c.client.PresignCreateDBCluster(ctx, input, presignOptFn) } func newServiceMetadataMiddleware_opCreateDBCluster(region string) *awsmiddleware.RegisterServiceMetadata { @@ -506,7 +511,7 @@ func newServiceMetadataMiddleware_opCreateDBCluster(region string) *awsmiddlewar // PresignCreateDBCluster is used to generate a presigned HTTP Request which // contains presigned URL, signed headers and HTTP method used. -func (c *PresignClient) PresignCreateDBCluster(ctx context.Context, params *CreateDBClusterInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { +func (c *PresignClient) PresignCreateDBCluster(ctx context.Context, params *CreateDBClusterInput, optFns ...func(*PresignOptions)) (*v4.PresignedHTTPRequest, error) { if params == nil { params = &CreateDBClusterInput{} } @@ -514,21 +519,24 @@ func (c *PresignClient) PresignCreateDBCluster(ctx context.Context, params *Crea for _, fn := range optFns { fn(&presignOptions) } - if presignOptions.Presigner != nil { - c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + if len(optFns) != 0 { + c = NewPresignClient(c.client, optFns...) } - clientOptFns := presignOptions.ClientOptions + + clientOptFns := make([]func(o *Options), 0) clientOptFns = append(clientOptFns, func(o *Options) { o.HTTPClient = &smithyhttp.NopClient{} }) + ctx = presignedurlcust.WithIsPresigning(ctx) result, _, err := c.client.invokeOperation(ctx, "CreateDBCluster", params, clientOptFns, addOperationCreateDBClusterMiddlewares, c.convertToPresignMiddleware, ) if err != nil { - return req, err + return nil, err } + out := result.(*v4.PresignedHTTPRequest) return out, nil } diff --git a/service/rds/api_op_CreateDBInstanceReadReplica.go b/service/rds/api_op_CreateDBInstanceReadReplica.go index be1147ffa59..5fef4758eba 100644 --- a/service/rds/api_op_CreateDBInstanceReadReplica.go +++ b/service/rds/api_op_CreateDBInstanceReadReplica.go @@ -475,34 +475,39 @@ func setCreateDBInstanceReadReplicadestinationRegion(params interface{}, value s input.destinationRegion = &value return nil } -func presignCreateDBInstanceReadReplica(ctx context.Context, client interface{}, region string, params interface{}) (req *v4.PresignedHTTPRequest, err error) { +func addCreateDBInstanceReadReplicaPresignURLMiddleware(stack *middleware.Stack, options Options) error { + return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ + Accessor: presignedurlcust.ParameterAccessor{ + GetPresignedURL: getCreateDBInstanceReadReplicaPreSignedUrl, + + GetSourceRegion: getCreateDBInstanceReadReplicaSourceRegion, + + CopyInput: copyCreateDBInstanceReadReplicaInputForPresign, + + SetDestinationRegion: setCreateDBInstanceReadReplicadestinationRegion, + + SetPresignedURL: setCreateDBInstanceReadReplicaPreSignedUrl, + }, + Presigner: &presignAutoFillCreateDBInstanceReadReplicaClient{client: NewPresignClient(New(options))}, + }) +} + +type presignAutoFillCreateDBInstanceReadReplicaClient struct { + client *PresignClient +} + +// PresignURL is a middleware accessor that satisfies URLPresigner interface. +func (c *presignAutoFillCreateDBInstanceReadReplicaClient) PresignURL(ctx context.Context, srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error) { input, ok := params.(*CreateDBInstanceReadReplicaInput) if !ok { - return req, fmt.Errorf("expect *CreateDBInstanceReadReplicaInput type, got %T", params) - } - c, ok := client.(*PresignClient) - if !ok { - return req, fmt.Errorf("expect *PresignClient type, got %T", client) + return nil, fmt.Errorf("expect *CreateDBInstanceReadReplicaInput type, got %T", params) } optFn := func(o *Options) { - o.Region = region + o.Region = srcRegion o.APIOptions = append(o.APIOptions, presignedurlcust.RemoveMiddleware) } presignOptFn := WithPresignClientFromClientOptions(optFn) - return c.PresignCreateDBInstanceReadReplica(ctx, input, presignOptFn) -} -func addCreateDBInstanceReadReplicaPresignURLMiddleware(stack *middleware.Stack, options Options) error { - return presignedurlcust.AddMiddleware(stack, presignedurlcust.Options{ - Accessor: presignedurlcust.ParameterAccessor{ - GetPresignedURL: getCreateDBInstanceReadReplicaPreSignedUrl, - GetSourceRegion: getCreateDBInstanceReadReplicaSourceRegion, - CopyInput: copyCreateDBInstanceReadReplicaInputForPresign, - SetDestinationRegion: setCreateDBInstanceReadReplicadestinationRegion, - SetPresignedURL: setCreateDBInstanceReadReplicaPreSignedUrl, - PresignOperation: presignCreateDBInstanceReadReplica, - }, - PresignClient: NewPresignClient(options), - }) + return c.client.PresignCreateDBInstanceReadReplica(ctx, input, presignOptFn) } func newServiceMetadataMiddleware_opCreateDBInstanceReadReplica(region string) *awsmiddleware.RegisterServiceMetadata { @@ -516,7 +521,7 @@ func newServiceMetadataMiddleware_opCreateDBInstanceReadReplica(region string) * // PresignCreateDBInstanceReadReplica is used to generate a presigned HTTP Request // which contains presigned URL, signed headers and HTTP method used. -func (c *PresignClient) PresignCreateDBInstanceReadReplica(ctx context.Context, params *CreateDBInstanceReadReplicaInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { +func (c *PresignClient) PresignCreateDBInstanceReadReplica(ctx context.Context, params *CreateDBInstanceReadReplicaInput, optFns ...func(*PresignOptions)) (*v4.PresignedHTTPRequest, error) { if params == nil { params = &CreateDBInstanceReadReplicaInput{} } @@ -524,21 +529,24 @@ func (c *PresignClient) PresignCreateDBInstanceReadReplica(ctx context.Context, for _, fn := range optFns { fn(&presignOptions) } - if presignOptions.Presigner != nil { - c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + if len(optFns) != 0 { + c = NewPresignClient(c.client, optFns...) } - clientOptFns := presignOptions.ClientOptions + + clientOptFns := make([]func(o *Options), 0) clientOptFns = append(clientOptFns, func(o *Options) { o.HTTPClient = &smithyhttp.NopClient{} }) + ctx = presignedurlcust.WithIsPresigning(ctx) result, _, err := c.client.invokeOperation(ctx, "CreateDBInstanceReadReplica", params, clientOptFns, addOperationCreateDBInstanceReadReplicaMiddlewares, c.convertToPresignMiddleware, ) if err != nil { - return req, err + return nil, err } + out := result.(*v4.PresignedHTTPRequest) return out, nil } diff --git a/service/rds/go.sum b/service/rds/go.sum index d39b60a4034..26564e07dc7 100644 --- a/service/rds/go.sum +++ b/service/rds/go.sum @@ -1,3 +1,10 @@ +github.com/aws/aws-sdk-go-v2 v0.28.0/go.mod h1:P9h1Cf+uOpElAT533QXKOzrpFaOlm8JMorThJNUfQ6Q= +github.com/aws/aws-sdk-go-v2 v0.29.0 h1:V/KKvuMO2hwHRg2SXJc5aasBHhD1AWbS6KMWg/Ueq1w= +github.com/aws/aws-sdk-go-v2 v0.29.0/go.mod h1:4d1/Ee0vCwCF7BfG1hCT3zu82493cRy5+VZ8JHvMPf0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.0 h1:r6FAnK2190ahYhtzj0HRcxCWlb7SE9/zI8v73npLkYU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.0/go.mod h1:aLwdZO0CArC1N9dbBxt7C2PBxkPktM0sDsr9iS7A9SY= +github.com/awslabs/smithy-go v0.2.1/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= +github.com/awslabs/smithy-go v0.3.0/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= github.com/awslabs/smithy-go v0.3.1-0.20201104233911-38864709e183/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4 h1:Aj5dOF+lDoEhU92no7YZF0IokuWGjiNrcm/DGIG3iII= github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= @@ -12,5 +19,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/service/s3/api_client.go b/service/s3/api_client.go index 8c6d60cdf54..67e23979c53 100644 --- a/service/s3/api_client.go +++ b/service/s3/api_client.go @@ -278,7 +278,8 @@ type PresignOptions struct { Presigner HTTPPresignerV4 // Expires sets the expiration duration for the generated presign url. This should - // be the duration in seconds the presigned URL should be considered valid for. + // be the duration in seconds the presigned URL should be considered valid for. If + // not set or set to zero, presign url would default to expire after 900 seconds. Expires time.Duration } @@ -313,48 +314,23 @@ type PresignClient struct { expires time.Duration } -func newPresignClient(client *Client, options PresignOptions) *PresignClient { - if options.Presigner == nil { - options.Presigner = v4.NewSigner() - } - return &PresignClient{ - client: client, - presigner: options.Presigner, - expires: options.Expires, - } -} - -// NewPresignClient generates a presign client using provided Client options and +// NewPresignClient generates a presign client using provided API Client and // presign options -func NewPresignClient(options Options, optFns ...func(*PresignOptions)) *PresignClient { - var presignOptions PresignOptions - for _, fn := range optFns { - fn(&presignOptions) - } - client := New(options, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) -} - -// NewPresignClientWrapper generates a presign client using provided API Client and -// presign options -func NewPresignClientWrapper(c *Client, optFns ...func(*PresignOptions)) *PresignClient { +func NewPresignClient(c *Client, optFns ...func(*PresignOptions)) *PresignClient { var presignOptions PresignOptions for _, fn := range optFns { fn(&presignOptions) } client := copyAPIClient(c, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) -} + if presignOptions.Presigner == nil { + presignOptions.Presigner = v4.NewSigner() + } -// NewPresignClientFromConfig generates a presign client using provided AWS config -// and presign options -func NewPresignClientFromConfig(cfg aws.Config, optFns ...func(*PresignOptions)) *PresignClient { - var presignOptions PresignOptions - for _, fn := range optFns { - fn(&presignOptions) + return &PresignClient{ + client: client, + presigner: presignOptions.Presigner, + expires: presignOptions.Expires, } - client := NewFromConfig(cfg, presignOptions.ClientOptions...) - return newPresignClient(client, presignOptions) } func copyAPIClient(c *Client, optFns ...func(*Options)) *Client { @@ -369,12 +345,11 @@ func (c *PresignClient) convertToPresignMiddleware(stack *middleware.Stack, opti if err != nil { return err } - if c.expires != 0 { - // add middleware to set expiration for s3 presigned url - err = stack.Build.Add(&s3cust.AddExpiresOnPresignedURL{Expires: c.expires}, middleware.After) - if err != nil { - return err - } + // add middleware to set expiration for s3 presigned url, if expiration is set to + // 0, this middleware sets a default expiration of 900 seconds + err = stack.Build.Add(&s3cust.AddExpiresOnPresignedURL{Expires: c.expires}, middleware.After) + if err != nil { + return err } return nil } diff --git a/service/s3/api_op_PutObject.go b/service/s3/api_op_PutObject.go index 2e758378e85..ada624cc11a 100644 --- a/service/s3/api_op_PutObject.go +++ b/service/s3/api_op_PutObject.go @@ -428,7 +428,7 @@ func addPutObjectUpdateEndpoint(stack *middleware.Stack, options Options) error // PresignPutObject is used to generate a presigned HTTP Request which contains // presigned URL, signed headers and HTTP method used. -func (c *PresignClient) PresignPutObject(ctx context.Context, params *PutObjectInput, optFns ...func(*PresignOptions)) (req *v4.PresignedHTTPRequest, err error) { +func (c *PresignClient) PresignPutObject(ctx context.Context, params *PutObjectInput, optFns ...func(*PresignOptions)) (*v4.PresignedHTTPRequest, error) { if params == nil { params = &PutObjectInput{} } @@ -436,21 +436,29 @@ func (c *PresignClient) PresignPutObject(ctx context.Context, params *PutObjectI for _, fn := range optFns { fn(&presignOptions) } - if presignOptions.Presigner != nil { - c = NewPresignClientWrapper(c.client, func(o *PresignOptions) { o.Presigner = presignOptions.Presigner }) + if len(optFns) != 0 { + c = NewPresignClient(c.client, optFns...) } - clientOptFns := presignOptions.ClientOptions + + clientOptFns := make([]func(o *Options), 0) clientOptFns = append(clientOptFns, func(o *Options) { o.HTTPClient = &smithyhttp.NopClient{} }) + ctx = presignedurlcust.WithIsPresigning(ctx) result, _, err := c.client.invokeOperation(ctx, "PutObject", params, clientOptFns, addOperationPutObjectMiddlewares, c.convertToPresignMiddleware, + addPutObjectPayloadAsUnsigned, ) if err != nil { - return req, err + return nil, err } + out := result.(*v4.PresignedHTTPRequest) return out, nil } + +func addPutObjectPayloadAsUnsigned(stack *middleware.Stack, options Options) error { + return v4.AddUnsignedPayloadMiddleware(stack) +} diff --git a/service/s3/go.sum b/service/s3/go.sum index e1dc3ec43bf..c8c9cef7f3d 100644 --- a/service/s3/go.sum +++ b/service/s3/go.sum @@ -1,3 +1,15 @@ +github.com/aws/aws-sdk-go-v2 v0.28.0/go.mod h1:P9h1Cf+uOpElAT533QXKOzrpFaOlm8JMorThJNUfQ6Q= +github.com/aws/aws-sdk-go-v2 v0.28.1-0.20201027160747-2592e9547fe4/go.mod h1:P9h1Cf+uOpElAT533QXKOzrpFaOlm8JMorThJNUfQ6Q= +github.com/aws/aws-sdk-go-v2 v0.29.0 h1:V/KKvuMO2hwHRg2SXJc5aasBHhD1AWbS6KMWg/Ueq1w= +github.com/aws/aws-sdk-go-v2 v0.29.0/go.mod h1:4d1/Ee0vCwCF7BfG1hCT3zu82493cRy5+VZ8JHvMPf0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v0.3.0 h1:OGNwNNeQvOZsa+zAK5nE7r6e0serfSAFznoXqbvbzFE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v0.3.0/go.mod h1:bMiNrEKNefchodwRJnuwaiAZj2NJq8ZHAYASve6mbFs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.0 h1:r6FAnK2190ahYhtzj0HRcxCWlb7SE9/zI8v73npLkYU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.0/go.mod h1:aLwdZO0CArC1N9dbBxt7C2PBxkPktM0sDsr9iS7A9SY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1-0.20201027184009-8eb8fc303e7c h1:Cm++f8Gc+n+R7LRF2/uYSxSRw+dIS3lepE7qCHsZzYE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1-0.20201027184009-8eb8fc303e7c/go.mod h1:/45/g9thJEPyLQ5x6wj5tlXz+i6uLWLlJkHABYdYFH0= +github.com/awslabs/smithy-go v0.2.1/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= +github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= github.com/awslabs/smithy-go v0.3.0/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= github.com/awslabs/smithy-go v0.3.1-0.20201104233911-38864709e183/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4 h1:Aj5dOF+lDoEhU92no7YZF0IokuWGjiNrcm/DGIG3iII= @@ -13,5 +25,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From fe3e6b5ede23fafd329607b953e7814df21418c0 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Mon, 16 Nov 2020 23:32:55 -0800 Subject: [PATCH 13/24] fix test --- service/internal/presigned-url/middleware_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/service/internal/presigned-url/middleware_test.go b/service/internal/presigned-url/middleware_test.go index 3af55ff4a0d..04c86a6a0b8 100644 --- a/service/internal/presigned-url/middleware_test.go +++ b/service/internal/presigned-url/middleware_test.go @@ -125,9 +125,8 @@ func getURLPresignMiddlewareOptions() Options { c.(*mockURLPresignInput).PresignedURL = v return nil }, - PresignOperation: presignURL, }, - PresignClient: mockURLPresigner{}, + Presigner: &mockURLPresigner{}, } } @@ -139,7 +138,7 @@ type mockURLPresignInput struct { type mockURLPresigner struct{} -func presignURL(ctx context.Context, client interface{}, srcRegion string, params interface{}) ( +func (*mockURLPresigner) PresignURL(ctx context.Context, srcRegion string, params interface{}) ( req *v4.PresignedHTTPRequest, err error, ) { in := params.(*mockURLPresignInput) From 78f6e3ec329d595226de821ae8b9efd8ea7754d7 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Tue, 17 Nov 2020 14:16:34 -0800 Subject: [PATCH 14/24] fix method that disables content sha256 computation from s3 streaming operation --- aws/signer/v4/middleware.go | 13 +++++++++++++ .../codegen/AwsHttpPresignURLClientGenerator.java | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/aws/signer/v4/middleware.go b/aws/signer/v4/middleware.go index 626715709ff..623399b5e3b 100644 --- a/aws/signer/v4/middleware.go +++ b/aws/signer/v4/middleware.go @@ -102,6 +102,13 @@ func AddComputePayloadSHA256Middleware(stack *middleware.Stack) error { return stack.Build.Add(&computePayloadSHA256{}, middleware.After) } +// RemoveComputePayloadSHA256Middleware removes computePayloadSHA256 from the +// operation middleware stack +func RemoveComputePayloadSHA256Middleware(stack *middleware.Stack) error { + _, err := stack.Build.Remove(computePayloadHashMiddlewareID) + return err +} + // ID is the middleware name func (m *computePayloadSHA256) ID() string { return computePayloadHashMiddlewareID @@ -159,6 +166,12 @@ func AddContentSHA256HeaderMiddleware(stack *middleware.Stack) error { return stack.Build.Insert(&contentSHA256Header{}, computePayloadHashMiddlewareID, middleware.After) } +// Removes ContentSHA256HeaderMiddleware from operation middleware stack +func RemoveContentSHA256HeaderMiddleware(stack *middleware.Stack) error { + _, err := stack.Build.Remove((*contentSHA256Header)(nil).ID()) + return err +} + // ID returns the ContentSHA256HeaderMiddleware identifier func (m *contentSHA256Header) ID() string { return "SigV4ContentSHA256Header" diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java index bd0c59da458..997337deafe 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java @@ -297,6 +297,10 @@ private void writeS3AddAsUnsignedPayloadHelper( addAsUnsignedPayloadName(operationSymbol.getName()), SymbolUtils.createPointableSymbolBuilder("Stack", SmithyGoDependency.SMITHY_MIDDLEWARE).build(), () -> { + writer.addUseImports(AwsGoDependency.AWS_SIGNER_V4); + writer.write("v4.RemoveContentSHA256HeaderMiddleware(stack)"); + writer.write("v4.RemoveComputePayloadSHA256Middleware(stack)"); + writer.write("return $T(stack)", SymbolUtils.createValueSymbolBuilder( "AddUnsignedPayloadMiddleware", AwsGoDependency.AWS_SIGNER_V4).build()); }); From ba5730ca0c5065e02dcd57b7179983fae197c532 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Tue, 17 Nov 2020 14:19:02 -0800 Subject: [PATCH 15/24] add handwritten presigning test for s3 --- .../internal/featuretest/s3/presign_test.go | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 service/internal/featuretest/s3/presign_test.go diff --git a/service/internal/featuretest/s3/presign_test.go b/service/internal/featuretest/s3/presign_test.go new file mode 100644 index 00000000000..53dc44c215f --- /dev/null +++ b/service/internal/featuretest/s3/presign_test.go @@ -0,0 +1,180 @@ +package unit_test + +import ( + "bytes" + "context" + "net/http" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/internal/awstesting/unit" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +func TestPutObject_PresignURL(t *testing.T) { + cases := map[string]struct { + input s3.PutObjectInput + options s3.PresignOptions + expectPresignedURLHost string + expectedRequestURILabel []string + expectSignedHeader http.Header + expectMethod string + expectError string + }{ + "standard case": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: strings.NewReader("hello-world"), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectedRequestURILabel: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "content-length": []string{"11"}, + "content-type": []string{"application/octet-stream"}, + "host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "seekable payload": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: bytes.NewReader([]byte("hello-world")), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectedRequestURILabel: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "content-length": []string{"11"}, + "content-type": []string{"application/octet-stream"}, + "host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "unseekable payload": { + // unseekable payload succeeds as we disable content sha256 computation for streaming input + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: bytes.NewBuffer([]byte(`hello-world`)), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectedRequestURILabel: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "content-length": []string{"11"}, + "content-type": []string{"application/octet-stream"}, + "host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "empty body": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: bytes.NewReader([]byte(``)), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectedRequestURILabel: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "content-type": {"application/octet-stream"}, + "host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "nil body": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectedRequestURILabel: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + cfg := aws.Config{ + Region: "us-west-2", + Credentials: unit.StubCredentialsProvider{}, + Retryer: aws.NoOpRetryer{}, + } + presignClient := s3.NewPresignClient(s3.NewFromConfig(cfg), func(options *s3.PresignOptions) { + options = &c.options + }) + + req, err := presignClient.PresignPutObject(ctx, &c.input) + if err != nil { + if len(c.expectError) == 0 { + t.Fatalf("expected no error, got %v", err) + } + // if expect error, match error and skip rest + if e, a := c.expectError, err.Error(); !strings.Contains(a, e) { + t.Fatalf("expected error to be %s, got %s", e, a) + } + } else { + if len(c.expectError) != 0 { + t.Fatalf("expected error to be %v, got none", c.expectError) + } + } + + if e, a := c.expectPresignedURLHost, req.URL; !strings.Contains(a, e) { + t.Fatalf("expected presigned url to contain host %s, got %s", e, a) + } + + if len(c.expectedRequestURILabel) != 0 { + for _, label := range c.expectedRequestURILabel { + if e, a := label, req.URL; !strings.Contains(a, e) { + t.Fatalf("expected presigned url to contain %v label in url: %v", label, req.URL) + } + } + } + + if e, a := c.expectSignedHeader, req.SignedHeader; len(cmp.Diff(e, a)) != 0 { + t.Fatalf("expected signed header to be %s, got %s, \n diff : %s", e, a, cmp.Diff(e, a)) + } + + if e, a := c.expectMethod, req.Method; !strings.EqualFold(e, a) { + t.Fatalf("expected presigning Method to be %s, got %s", e, a) + } + + }) + } +} From 0d0e272ad694eb347d2363afdf9ec84889b52b51 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Tue, 17 Nov 2020 14:19:25 -0800 Subject: [PATCH 16/24] regenerated s3 client --- service/s3/api_op_PutObject.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/s3/api_op_PutObject.go b/service/s3/api_op_PutObject.go index ada624cc11a..8be377248af 100644 --- a/service/s3/api_op_PutObject.go +++ b/service/s3/api_op_PutObject.go @@ -460,5 +460,7 @@ func (c *PresignClient) PresignPutObject(ctx context.Context, params *PutObjectI } func addPutObjectPayloadAsUnsigned(stack *middleware.Stack, options Options) error { + v4.RemoveContentSHA256HeaderMiddleware(stack) + v4.RemoveComputePayloadSHA256Middleware(stack) return v4.AddUnsignedPayloadMiddleware(stack) } From 0574c9c2e731a1613ca63f2649071c48d4aa21a5 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Tue, 17 Nov 2020 14:27:42 -0800 Subject: [PATCH 17/24] add go mod for feature test module --- service/internal/featuretest/go.mod | 19 +++++++++++++++++++ service/internal/featuretest/go.sum | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 service/internal/featuretest/go.mod create mode 100644 service/internal/featuretest/go.sum diff --git a/service/internal/featuretest/go.mod b/service/internal/featuretest/go.mod new file mode 100644 index 00000000000..b0a3c2e73aa --- /dev/null +++ b/service/internal/featuretest/go.mod @@ -0,0 +1,19 @@ +module github.com/aws/aws-sdk-go-v2/service/internal/unittest + +go 1.15 + +require ( + github.com/aws/aws-sdk-go-v2 v0.29.0 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v0.29.0 + github.com/google/go-cmp v0.5.2 +) + +replace ( + github.com/aws/aws-sdk-go-v2 => ../../../ + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../service/internal/accept-encoding/ + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../service/internal/presigned-url/ + github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../service/internal/s3shared/ + github.com/aws/aws-sdk-go-v2/service/s3 => ../../../service/s3/ +) diff --git a/service/internal/featuretest/go.sum b/service/internal/featuretest/go.sum new file mode 100644 index 00000000000..6a38437d0ac --- /dev/null +++ b/service/internal/featuretest/go.sum @@ -0,0 +1,18 @@ +github.com/awslabs/smithy-go v0.3.0 h1:I1EQ1P+VtxpuNnGYymATewaKrlnaYQwFvO8lNTsafbs= +github.com/awslabs/smithy-go v0.3.0/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= +github.com/awslabs/smithy-go v0.3.1-0.20201104233911-38864709e183/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= +github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4 h1:Aj5dOF+lDoEhU92no7YZF0IokuWGjiNrcm/DGIG3iII= +github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 5844f2c7021b2db930b887a2b44615d56947bbdb Mon Sep 17 00:00:00 2001 From: skotambkar Date: Tue, 17 Nov 2020 16:22:59 -0800 Subject: [PATCH 18/24] add presign integration test --- .../integrationtest/s3/presign_test.go | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 service/internal/integrationtest/s3/presign_test.go diff --git a/service/internal/integrationtest/s3/presign_test.go b/service/internal/integrationtest/s3/presign_test.go new file mode 100644 index 00000000000..2bdfa1a104b --- /dev/null +++ b/service/internal/integrationtest/s3/presign_test.go @@ -0,0 +1,73 @@ +// +build integration + +package s3 + +func TestInteg_PresignURL_PutObject(t *testing.T) { + + ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelFn() + + cfg, err := integrationtest.LoadConfigWithDefaultRegion("us-west-2") + if err != nil { + t.Fatalf("failed to load config, %v", err) + } + + client := s3.NewFromConfig(cfg) + + bucketName := "mockbucket-01" + key := "random" + + params := &s3.PutObjectInput{ + Bucket: &bucketName, + Key: &key, + Body: bytes.NewReader([]byte(`Hello-world`)), + } + + presignerClient := s3.NewPresignClient(client, func(options *s3.PresignOptions) { + options.Expires = 600 * time.Second + }) + + presignRequest, err := presignerClient.PresignPutObject(ctx, params) + if err != nil { + t.Errorf("expect no error, got %v", err) + } + + // Putobject + t.Logf("url : %v \n", presignRequest.URL) + t.Logf("method : %v \n", presignRequest.Method) + t.Logf("signed headers : %v \n", presignRequest.SignedHeader) + + t.Logf("attempting to put request") + + req, err := http.NewRequest(presignRequest.Method, presignRequest.URL, nil) + if err != nil { + t.Fatalf("failed to build presigned request, %v", err) + } + + for k, vs := range presignRequest.SignedHeader { + for _, v := range vs { + req.Header.Add(k, v) + } + } + + // Need to ensure that the content length member is set of the HTTP Request + // or the request will not be transmitted correctly with a content length + // value across the wire. + if contLen := req.Header.Get("Content-Length"); len(contLen) > 0 { + req.ContentLength, _ = strconv.ParseInt(contLen, 10, 64) + } + + req.Body = ioutil.NopCloser(params.Body) + + // Upload the file contents to S3. + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("failed to do PUT request, %v", err) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("failed to put S3 object, %d:%s", resp.StatusCode, resp.Status) + } +} From 5e56d2b4885f9033724f75269a3dfc9d6cba2a71 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Tue, 17 Nov 2020 17:03:43 -0800 Subject: [PATCH 19/24] adds integration test for presign url and moves the unit test to service/s3 folder --- service/internal/featuretest/go.mod | 19 --------- service/internal/featuretest/go.sum | 18 --------- .../integrationtest/s3/presign_test.go | 40 ++++++++++++------- .../internal/customizations}/presign_test.go | 2 +- 4 files changed, 26 insertions(+), 53 deletions(-) delete mode 100644 service/internal/featuretest/go.mod delete mode 100644 service/internal/featuretest/go.sum rename service/{internal/featuretest/s3 => s3/internal/customizations}/presign_test.go (99%) diff --git a/service/internal/featuretest/go.mod b/service/internal/featuretest/go.mod deleted file mode 100644 index b0a3c2e73aa..00000000000 --- a/service/internal/featuretest/go.mod +++ /dev/null @@ -1,19 +0,0 @@ -module github.com/aws/aws-sdk-go-v2/service/internal/unittest - -go 1.15 - -require ( - github.com/aws/aws-sdk-go-v2 v0.29.0 - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v0.29.0 - github.com/google/go-cmp v0.5.2 -) - -replace ( - github.com/aws/aws-sdk-go-v2 => ../../../ - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../service/internal/accept-encoding/ - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../service/internal/presigned-url/ - github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../service/internal/s3shared/ - github.com/aws/aws-sdk-go-v2/service/s3 => ../../../service/s3/ -) diff --git a/service/internal/featuretest/go.sum b/service/internal/featuretest/go.sum deleted file mode 100644 index 6a38437d0ac..00000000000 --- a/service/internal/featuretest/go.sum +++ /dev/null @@ -1,18 +0,0 @@ -github.com/awslabs/smithy-go v0.3.0 h1:I1EQ1P+VtxpuNnGYymATewaKrlnaYQwFvO8lNTsafbs= -github.com/awslabs/smithy-go v0.3.0/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= -github.com/awslabs/smithy-go v0.3.1-0.20201104233911-38864709e183/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= -github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4 h1:Aj5dOF+lDoEhU92no7YZF0IokuWGjiNrcm/DGIG3iII= -github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/service/internal/integrationtest/s3/presign_test.go b/service/internal/integrationtest/s3/presign_test.go index 2bdfa1a104b..76c90b42d1d 100644 --- a/service/internal/integrationtest/s3/presign_test.go +++ b/service/internal/integrationtest/s3/presign_test.go @@ -2,7 +2,22 @@ package s3 +import ( + "bytes" + "context" + "io/ioutil" + "net/http" + "strconv" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/internal/integrationtest" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + func TestInteg_PresignURL_PutObject(t *testing.T) { + key := integrationtest.UniqueID() ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFn() @@ -14,12 +29,9 @@ func TestInteg_PresignURL_PutObject(t *testing.T) { client := s3.NewFromConfig(cfg) - bucketName := "mockbucket-01" - key := "random" - params := &s3.PutObjectInput{ - Bucket: &bucketName, - Key: &key, + Bucket: &setupMetadata.Buckets.Source.Name, + Key: aws.String(key), Body: bytes.NewReader([]byte(`Hello-world`)), } @@ -32,18 +44,13 @@ func TestInteg_PresignURL_PutObject(t *testing.T) { t.Errorf("expect no error, got %v", err) } - // Putobject - t.Logf("url : %v \n", presignRequest.URL) - t.Logf("method : %v \n", presignRequest.Method) - t.Logf("signed headers : %v \n", presignRequest.SignedHeader) - - t.Logf("attempting to put request") - + // create a http request req, err := http.NewRequest(presignRequest.Method, presignRequest.URL, nil) if err != nil { t.Fatalf("failed to build presigned request, %v", err) } + // assign the signed headers onto the http request for k, vs := range presignRequest.SignedHeader { for _, v := range vs { req.Header.Add(k, v) @@ -51,15 +58,18 @@ func TestInteg_PresignURL_PutObject(t *testing.T) { } // Need to ensure that the content length member is set of the HTTP Request - // or the request will not be transmitted correctly with a content length + // or the request will NOT be transmitted correctly with a content length // value across the wire. if contLen := req.Header.Get("Content-Length"); len(contLen) > 0 { req.ContentLength, _ = strconv.ParseInt(contLen, 10, 64) } - req.Body = ioutil.NopCloser(params.Body) + // assign the request body if not nil + if params.Body != nil { + req.Body = ioutil.NopCloser(params.Body) + } - // Upload the file contents to S3. + // Upload the object to S3. resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("failed to do PUT request, %v", err) diff --git a/service/internal/featuretest/s3/presign_test.go b/service/s3/internal/customizations/presign_test.go similarity index 99% rename from service/internal/featuretest/s3/presign_test.go rename to service/s3/internal/customizations/presign_test.go index 53dc44c215f..d64175c87ce 100644 --- a/service/internal/featuretest/s3/presign_test.go +++ b/service/s3/internal/customizations/presign_test.go @@ -1,4 +1,4 @@ -package unit_test +package customizations_test import ( "bytes" From a6b12b5e13b07178b12a826b365e26adc1fc3613 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Tue, 17 Nov 2020 18:18:42 -0800 Subject: [PATCH 20/24] revert the integration plugin order change --- ...azon.smithy.go.codegen.integration.GoIntegration | 6 +++--- service/s3/api_client.go | 1 + service/s3/api_op_PutObject.go | 2 +- service/s3/go.mod | 3 ++- service/s3/go.sum | 13 ------------- 5 files changed, 7 insertions(+), 18 deletions(-) diff --git a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index 5a88e0f1191..7e1dfee480e 100644 --- a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -10,9 +10,6 @@ software.amazon.smithy.aws.go.codegen.AwsSdkServiceId software.amazon.smithy.aws.go.codegen.AwsRetryMiddlewareHelper software.amazon.smithy.aws.go.codegen.AWSRequestIDRetriever software.amazon.smithy.aws.go.codegen.AWSResponseErrorWrapper -software.amazon.smithy.aws.go.codegen.AwsHttpPresignURLClientGenerator -software.amazon.smithy.aws.go.codegen.FilterStreamingOperations -software.amazon.smithy.aws.go.codegen.RequestResponseLogging software.amazon.smithy.aws.go.codegen.customization.DynamoDBValidateResponseChecksum software.amazon.smithy.aws.go.codegen.customization.S3UpdateEndpoint software.amazon.smithy.aws.go.codegen.customization.APIGatewayAcceptHeader @@ -29,4 +26,7 @@ software.amazon.smithy.aws.go.codegen.customization.S3ErrorWith200Status software.amazon.smithy.aws.go.codegen.customization.Route53Customizations software.amazon.smithy.aws.go.codegen.customization.S3HeadObjectCustomizations software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill +software.amazon.smithy.aws.go.codegen.FilterStreamingOperations +software.amazon.smithy.aws.go.codegen.RequestResponseLogging software.amazon.smithy.aws.go.codegen.customization.S3ControlEndpointResolver +software.amazon.smithy.aws.go.codegen.AwsHttpPresignURLClientGenerator diff --git a/service/s3/api_client.go b/service/s3/api_client.go index 67e23979c53..b52d80b46d0 100644 --- a/service/s3/api_client.go +++ b/service/s3/api_client.go @@ -11,6 +11,7 @@ import ( awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" acceptencodingcust "github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding" "github.com/aws/aws-sdk-go-v2/service/internal/s3shared" + s3cust "github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations" smithy "github.com/awslabs/smithy-go" "github.com/awslabs/smithy-go/logging" "github.com/awslabs/smithy-go/middleware" diff --git a/service/s3/api_op_PutObject.go b/service/s3/api_op_PutObject.go index 8be377248af..8e5f4c92723 100644 --- a/service/s3/api_op_PutObject.go +++ b/service/s3/api_op_PutObject.go @@ -6,8 +6,8 @@ import ( "context" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" - s3cust "github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations" presignedurlcust "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url" + s3cust "github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" diff --git a/service/s3/go.mod b/service/s3/go.mod index 08f15b1aa29..1b09d08a705 100644 --- a/service/s3/go.mod +++ b/service/s3/go.mod @@ -5,9 +5,10 @@ go 1.15 require ( github.com/aws/aws-sdk-go-v2 v0.29.1-0.20201113222241-726e4a15683d github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v0.3.1-0.20201113222241-726e4a15683d + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.1-0.20201113222241-726e4a15683d github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1-0.20201113222241-726e4a15683d - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.0 github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4 + github.com/google/go-cmp v0.5.2 ) replace github.com/aws/aws-sdk-go-v2 => ../../ diff --git a/service/s3/go.sum b/service/s3/go.sum index c8c9cef7f3d..e1dc3ec43bf 100644 --- a/service/s3/go.sum +++ b/service/s3/go.sum @@ -1,15 +1,3 @@ -github.com/aws/aws-sdk-go-v2 v0.28.0/go.mod h1:P9h1Cf+uOpElAT533QXKOzrpFaOlm8JMorThJNUfQ6Q= -github.com/aws/aws-sdk-go-v2 v0.28.1-0.20201027160747-2592e9547fe4/go.mod h1:P9h1Cf+uOpElAT533QXKOzrpFaOlm8JMorThJNUfQ6Q= -github.com/aws/aws-sdk-go-v2 v0.29.0 h1:V/KKvuMO2hwHRg2SXJc5aasBHhD1AWbS6KMWg/Ueq1w= -github.com/aws/aws-sdk-go-v2 v0.29.0/go.mod h1:4d1/Ee0vCwCF7BfG1hCT3zu82493cRy5+VZ8JHvMPf0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v0.3.0 h1:OGNwNNeQvOZsa+zAK5nE7r6e0serfSAFznoXqbvbzFE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v0.3.0/go.mod h1:bMiNrEKNefchodwRJnuwaiAZj2NJq8ZHAYASve6mbFs= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.0 h1:r6FAnK2190ahYhtzj0HRcxCWlb7SE9/zI8v73npLkYU= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v0.1.0/go.mod h1:aLwdZO0CArC1N9dbBxt7C2PBxkPktM0sDsr9iS7A9SY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1-0.20201027184009-8eb8fc303e7c h1:Cm++f8Gc+n+R7LRF2/uYSxSRw+dIS3lepE7qCHsZzYE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v0.3.1-0.20201027184009-8eb8fc303e7c/go.mod h1:/45/g9thJEPyLQ5x6wj5tlXz+i6uLWLlJkHABYdYFH0= -github.com/awslabs/smithy-go v0.2.1/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= -github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= github.com/awslabs/smithy-go v0.3.0/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= github.com/awslabs/smithy-go v0.3.1-0.20201104233911-38864709e183/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI= github.com/awslabs/smithy-go v0.3.1-0.20201108010311-62c2a93810b4 h1:Aj5dOF+lDoEhU92no7YZF0IokuWGjiNrcm/DGIG3iII= @@ -25,6 +13,5 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From cee7c26a314c19f1172ac3e64978d8e4a6047d7b Mon Sep 17 00:00:00 2001 From: skotambkar Date: Tue, 17 Nov 2020 22:20:39 -0800 Subject: [PATCH 21/24] go lint fix --- aws/signer/v4/middleware.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aws/signer/v4/middleware.go b/aws/signer/v4/middleware.go index 623399b5e3b..272baf37e6d 100644 --- a/aws/signer/v4/middleware.go +++ b/aws/signer/v4/middleware.go @@ -166,7 +166,8 @@ func AddContentSHA256HeaderMiddleware(stack *middleware.Stack) error { return stack.Build.Insert(&contentSHA256Header{}, computePayloadHashMiddlewareID, middleware.After) } -// Removes ContentSHA256HeaderMiddleware from operation middleware stack +// RemoveContentSHA256HeaderMiddleware removes contentSHA256Header middleware +// from the operation middleware stack func RemoveContentSHA256HeaderMiddleware(stack *middleware.Stack) error { _, err := stack.Build.Remove((*contentSHA256Header)(nil).ID()) return err From e4df8b60e4d0e6aa8f2bb1ffdb674e72f44e5baa Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 18 Nov 2020 23:43:04 -0800 Subject: [PATCH 22/24] add support for s3 GetObject presigned url --- .../AwsHttpPresignURLClientGenerator.java | 5 ++- service/s3/api_op_GetObject.go | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java index 997337deafe..eb0981038a7 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java @@ -80,7 +80,10 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration { // constant map with service to list of operation for which presignedURL client and operation must be generated. private static final Map> presignedClientMap = MapUtils.of( - ShapeId.from("com.amazonaws.s3#AmazonS3"), SetUtils.of(ShapeId.from("com.amazonaws.s3#PutObject")) + ShapeId.from("com.amazonaws.s3#AmazonS3"), SetUtils.of( + ShapeId.from("com.amazonaws.s3#GetObject"), + ShapeId.from("com.amazonaws.s3#PutObject") + ) ); private static final String addAsUnsignedPayloadName(String operationName) { diff --git a/service/s3/api_op_GetObject.go b/service/s3/api_op_GetObject.go index 76a279a0d6d..cb641f6ef04 100644 --- a/service/s3/api_op_GetObject.go +++ b/service/s3/api_op_GetObject.go @@ -6,6 +6,7 @@ import ( "context" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + presignedurlcust "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url" s3cust "github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/awslabs/smithy-go/middleware" @@ -473,3 +474,35 @@ func addGetObjectUpdateEndpoint(stack *middleware.Stack, options Options) error UseARNRegion: options.UseARNRegion, }) } + +// PresignGetObject is used to generate a presigned HTTP Request which contains +// presigned URL, signed headers and HTTP method used. +func (c *PresignClient) PresignGetObject(ctx context.Context, params *GetObjectInput, optFns ...func(*PresignOptions)) (*v4.PresignedHTTPRequest, error) { + if params == nil { + params = &GetObjectInput{} + } + var presignOptions PresignOptions + for _, fn := range optFns { + fn(&presignOptions) + } + if len(optFns) != 0 { + c = NewPresignClient(c.client, optFns...) + } + + clientOptFns := make([]func(o *Options), 0) + clientOptFns = append(clientOptFns, func(o *Options) { + o.HTTPClient = &smithyhttp.NopClient{} + }) + + ctx = presignedurlcust.WithIsPresigning(ctx) + result, _, err := c.client.invokeOperation(ctx, "GetObject", params, clientOptFns, + addOperationGetObjectMiddlewares, + c.convertToPresignMiddleware, + ) + if err != nil { + return nil, err + } + + out := result.(*v4.PresignedHTTPRequest) + return out, nil +} From b542d1be495b71ae3a592573e758bab7b58d3e5c Mon Sep 17 00:00:00 2001 From: skotambkar Date: Wed, 18 Nov 2020 23:53:06 -0800 Subject: [PATCH 23/24] updates integ test and unit test --- .../integrationtest/s3/presign_test.go | 145 +++++++++++++----- .../internal/customizations/presign_test.go | 28 ++-- 2 files changed, 122 insertions(+), 51 deletions(-) diff --git a/service/internal/integrationtest/s3/presign_test.go b/service/internal/integrationtest/s3/presign_test.go index 76c90b42d1d..9aa8e4483d2 100644 --- a/service/internal/integrationtest/s3/presign_test.go +++ b/service/internal/integrationtest/s3/presign_test.go @@ -5,49 +5,128 @@ package s3 import ( "bytes" "context" + "fmt" + "io" "io/ioutil" "net/http" "strconv" "testing" "time" - "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/aws-sdk-go-v2/service/internal/integrationtest" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/go-cmp/cmp" ) -func TestInteg_PresignURL_PutObject(t *testing.T) { - key := integrationtest.UniqueID() - - ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) - defer cancelFn() - - cfg, err := integrationtest.LoadConfigWithDefaultRegion("us-west-2") - if err != nil { - t.Fatalf("failed to load config, %v", err) +func TestInteg_PresignURL(t *testing.T) { + cases := map[string]struct { + body io.Reader + expires time.Duration + sha256Header string + expectedSignedHeader http.Header + }{ + "standard": { + body: bytes.NewReader([]byte("Hello-world")), + expectedSignedHeader: http.Header{ + "content-type": {"application/octet-stream"}, + "content-length": {"11"}, + }, + }, + "nil-body": { + expectedSignedHeader: http.Header{}, + }, + // TODO: fix PutObject with an empty body returning 501 + // "empty-body": { + // body: bytes.NewReader([]byte("")), + // expectedSignedHeader: http.Header{ + // "content-type": {"application/octet-stream"}, + // }, + // }, } - client := s3.NewFromConfig(cfg) - - params := &s3.PutObjectInput{ - Bucket: &setupMetadata.Buckets.Source.Name, - Key: aws.String(key), - Body: bytes.NewReader([]byte(`Hello-world`)), - } - - presignerClient := s3.NewPresignClient(client, func(options *s3.PresignOptions) { - options.Expires = 600 * time.Second - }) - - presignRequest, err := presignerClient.PresignPutObject(ctx, params) - if err != nil { - t.Errorf("expect no error, got %v", err) + for name, c := range cases { + t.Run(name, func(t *testing.T) { + key := integrationtest.UniqueID() + + ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelFn() + + cfg, err := integrationtest.LoadConfigWithDefaultRegion("us-west-2") + if err != nil { + t.Fatalf("failed to load config, %v", err) + } + + client := s3.NewFromConfig(cfg) + + // construct a put object + putObjectInput := &s3.PutObjectInput{ + Bucket: &setupMetadata.Buckets.Source.Name, + Key: &key, + Body: c.body, + } + + presignerClient := s3.NewPresignClient(client, func(options *s3.PresignOptions) { + options.Expires = 600 * time.Second + }) + + presignRequest, err := presignerClient.PresignPutObject(ctx, putObjectInput) + if err != nil { + t.Errorf("expect no error, got %v", err) + } + + for k, v := range c.expectedSignedHeader { + value := presignRequest.SignedHeader[k] + if len(value) == 0 { + t.Fatalf("expected %v header to be present in presigned url, got %v", k, presignRequest.SignedHeader) + } + + if diff := cmp.Diff(v, value); len(diff) != 0 { + t.Fatalf("expected %v header value to be %v got %v", k, v, value) + } + } + + resp, err := sendHTTPRequest(presignRequest, putObjectInput.Body) + if err != nil { + t.Errorf("expect no error while sending HTTP request using presigned url, got %v", err) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("failed to put S3 object, %d:%s", resp.StatusCode, resp.Status) + } + + // construct a get object + getObjectInput := &s3.GetObjectInput{ + Bucket: &setupMetadata.Buckets.Source.Name, + Key: &key, + } + + presignRequest, err = presignerClient.PresignGetObject(ctx, getObjectInput) + if err != nil { + t.Errorf("expect no error, got %v", err) + } + + resp, err = sendHTTPRequest(presignRequest, nil) + if err != nil { + t.Errorf("expect no error while sending HTTP request using presigned url, got %v", err) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("failed to get S3 object, %d:%s", resp.StatusCode, resp.Status) + } + }) } +} +func sendHTTPRequest(presignRequest *v4.PresignedHTTPRequest, body io.Reader) (*http.Response, error) { // create a http request req, err := http.NewRequest(presignRequest.Method, presignRequest.URL, nil) if err != nil { - t.Fatalf("failed to build presigned request, %v", err) + return nil, fmt.Errorf("failed to build presigned request, %v", err) } // assign the signed headers onto the http request @@ -65,19 +144,11 @@ func TestInteg_PresignURL_PutObject(t *testing.T) { } // assign the request body if not nil - if params.Body != nil { - req.Body = ioutil.NopCloser(params.Body) + if body != nil { + req.Body = ioutil.NopCloser(body) } // Upload the object to S3. resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("failed to do PUT request, %v", err) - } - - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - t.Fatalf("failed to put S3 object, %d:%s", resp.StatusCode, resp.Status) - } + return resp, err } diff --git a/service/s3/internal/customizations/presign_test.go b/service/s3/internal/customizations/presign_test.go index d64175c87ce..f67df2f5d5e 100644 --- a/service/s3/internal/customizations/presign_test.go +++ b/service/s3/internal/customizations/presign_test.go @@ -16,13 +16,13 @@ import ( func TestPutObject_PresignURL(t *testing.T) { cases := map[string]struct { - input s3.PutObjectInput - options s3.PresignOptions - expectPresignedURLHost string - expectedRequestURILabel []string - expectSignedHeader http.Header - expectMethod string - expectError string + input s3.PutObjectInput + options s3.PresignOptions + expectPresignedURLHost string + expectRequestURIQuery []string + expectSignedHeader http.Header + expectMethod string + expectError string }{ "standard case": { input: s3.PutObjectInput{ @@ -31,7 +31,7 @@ func TestPutObject_PresignURL(t *testing.T) { Body: strings.NewReader("hello-world"), }, expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", - expectedRequestURILabel: []string{ + expectRequestURIQuery: []string{ "X-Amz-Expires=900", "X-Amz-Credential", "X-Amz-Date", @@ -52,7 +52,7 @@ func TestPutObject_PresignURL(t *testing.T) { Body: bytes.NewReader([]byte("hello-world")), }, expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", - expectedRequestURILabel: []string{ + expectRequestURIQuery: []string{ "X-Amz-Expires=900", "X-Amz-Credential", "X-Amz-Date", @@ -74,7 +74,7 @@ func TestPutObject_PresignURL(t *testing.T) { Body: bytes.NewBuffer([]byte(`hello-world`)), }, expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", - expectedRequestURILabel: []string{ + expectRequestURIQuery: []string{ "X-Amz-Expires=900", "X-Amz-Credential", "X-Amz-Date", @@ -95,7 +95,7 @@ func TestPutObject_PresignURL(t *testing.T) { Body: bytes.NewReader([]byte(``)), }, expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", - expectedRequestURILabel: []string{ + expectRequestURIQuery: []string{ "X-Amz-Expires=900", "X-Amz-Credential", "X-Amz-Date", @@ -114,7 +114,7 @@ func TestPutObject_PresignURL(t *testing.T) { Key: aws.String("mockkey"), }, expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", - expectedRequestURILabel: []string{ + expectRequestURIQuery: []string{ "X-Amz-Expires=900", "X-Amz-Credential", "X-Amz-Date", @@ -159,8 +159,8 @@ func TestPutObject_PresignURL(t *testing.T) { t.Fatalf("expected presigned url to contain host %s, got %s", e, a) } - if len(c.expectedRequestURILabel) != 0 { - for _, label := range c.expectedRequestURILabel { + if len(c.expectRequestURIQuery) != 0 { + for _, label := range c.expectRequestURIQuery { if e, a := label, req.URL; !strings.Contains(a, e) { t.Fatalf("expected presigned url to contain %v label in url: %v", label, req.URL) } From 8db7d37cd0624321d615a9ad626f3564d7787ffd Mon Sep 17 00:00:00 2001 From: skotambkar Date: Thu, 19 Nov 2020 22:37:05 -0800 Subject: [PATCH 24/24] enable empty body test for presign --- service/internal/integrationtest/go.mod | 1 + .../integrationtest/s3/presign_test.go | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/service/internal/integrationtest/go.mod b/service/internal/integrationtest/go.mod index eac1fcefbfb..3b835ba95d0 100644 --- a/service/internal/integrationtest/go.mod +++ b/service/internal/integrationtest/go.mod @@ -87,6 +87,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/wafv2 v0.29.0 github.com/aws/aws-sdk-go-v2/service/workspaces v0.29.0 github.com/awslabs/smithy-go v0.3.1-0.20201120010914-c02b9493fe20 + github.com/google/go-cmp v0.5.2 ) go 1.15 diff --git a/service/internal/integrationtest/s3/presign_test.go b/service/internal/integrationtest/s3/presign_test.go index 9aa8e4483d2..027329cd383 100644 --- a/service/internal/integrationtest/s3/presign_test.go +++ b/service/internal/integrationtest/s3/presign_test.go @@ -36,13 +36,12 @@ func TestInteg_PresignURL(t *testing.T) { "nil-body": { expectedSignedHeader: http.Header{}, }, - // TODO: fix PutObject with an empty body returning 501 - // "empty-body": { - // body: bytes.NewReader([]byte("")), - // expectedSignedHeader: http.Header{ - // "content-type": {"application/octet-stream"}, - // }, - // }, + "empty-body": { + body: bytes.NewReader([]byte("")), + expectedSignedHeader: http.Header{ + "content-type": {"application/octet-stream"}, + }, + }, } for name, c := range cases { @@ -72,7 +71,7 @@ func TestInteg_PresignURL(t *testing.T) { presignRequest, err := presignerClient.PresignPutObject(ctx, putObjectInput) if err != nil { - t.Errorf("expect no error, got %v", err) + t.Fatalf("expect no error, got %v", err) } for k, v := range c.expectedSignedHeader { @@ -146,6 +145,9 @@ func sendHTTPRequest(presignRequest *v4.PresignedHTTPRequest, body io.Reader) (* // assign the request body if not nil if body != nil { req.Body = ioutil.NopCloser(body) + if req.ContentLength == 0 { + req.Body = nil + } } // Upload the object to S3.