From 2592e9547fe4845087077ed752c08b0e7425ac2a Mon Sep 17 00:00:00 2001 From: Jason Del Ponte Date: Wed, 21 Oct 2020 14:20:16 -0700 Subject: [PATCH] codegen: Add flag to direct if SDK is allowed to modify a custom endpoint or not Adds a new member to the `aws.Endpoint` struct, `HostnameImmutable`. This member directs if the SDK is allowed to modify the resolved endpoint to meet the requirements of the API. If `HostnameImmutable` is true, the SDK will not attempt to modify the hostname via any customizations, or prefix behavior. Flag defaults to false. - Fixes https://github.com/aws/aws-sdk-go-v2/issues/827 (SDK requiring region for endpoint resolution) - Fixes https://github.com/aws/aws-sdk-go-v2/issues/328 - Fixes https://github.com/aws/aws-sdk-go-v2/issues/364 - Related to https://github.com/aws/aws-sdk-go-v2/pull/836 When Amazon S3 Accesspoint support is implemented. - Related to https://github.com/aws/aws-sdk-go-v2/issues/479 Depends on https://github.com/awslabs/smithy-go/pull/222 --- aws/endpoints.go | 58 +++- aws/errors.go | 12 +- .../aws/go/codegen/EndpointGenerator.java | 19 +- .../customization/presign_url.go.template | 2 +- feature/s3/manager/bucket_region_test.go | 4 +- feature/s3/manager/download_test.go | 4 +- .../s3/manager/internal/testing/endpoints.go | 4 +- feature/s3/manager/upload_test.go | 3 +- service/internal/s3shared/update_endpoint.go | 8 +- service/s3/internal/customizations/doc.go | 6 + .../customizations/handle_200_error_test.go | 6 +- .../customizations/update_endpoint.go | 22 +- .../customizations/update_endpoint_test.go | 296 ++++++++++++------ .../customizations/update_endpoint_test.go | 156 ++++++--- 14 files changed, 412 insertions(+), 188 deletions(-) diff --git a/aws/endpoints.go b/aws/endpoints.go index 859ecf5c065..fdea51add19 100644 --- a/aws/endpoints.go +++ b/aws/endpoints.go @@ -4,12 +4,44 @@ import ( "fmt" ) -// Endpoint represents the endpoint a service client should make requests to. +// Endpoint represents the endpoint a service client should make API operation +// calls to. +// +// The SDK will automatically resolve these endpoints per API client using an +// internal endpoint resolvers. If you'd like to provide custom endpoint +// resolving behavior you can implement the EndpointResolver interface. type Endpoint struct { - // The URL of the endpoint. + // The base URL endpoint the SDK API clients will use to make API calls to. + // The SDK will suffix URI path and query elements to this endpoint. URL string - // The endpoint partition + // Specifies if the endpoint's hostname can be modified by the SDK's API + // client. + // + // If the hostname is mutable the SDK API clients may modify any part of + // the hostname based on the requirements of the API, (e.g. adding, or + // removing content in the hostname). Such as, Amazon S3 API client + // prefixing "bucketname" to the hostname, or changing the + // hostname service name component from "s3." to "s3-accesspoint.dualstack." + // for the dualstack endpoint of an S3 Accesspoint resource. + // + // Care should be taken when providing a custom endpoint for an API. If the + // endpoint hostname is mutable, and the client cannot modify the endpoint + // correctly, the operation call will most likely fail, or have undefined + // behavior. + // + // If hostname is immutable, the SDK API clients will not modify the + // hostname of the URL. This may cause the API client not to function + // correctly if the API requires the operation specific hostname values + // to be used by the client. + // + // This flag does not modify the API client's behavior if this endpoint + // will be used instead of Endpoint Discovery, or if the endpoint will be + // used to perform Endpoint Discovery. That behavior is configured via the + // API Client's Options. + HostnameImmutable bool + + // The AWS partition the endpoint belongs to. PartitionID string // The service name that should be used for signing the requests to the @@ -24,9 +56,11 @@ type Endpoint struct { SigningMethod string } -// EndpointNotFoundError is a sentinel error to indicate that the EndpointResolver implementation was unable -// to resolve an endpoint for the given service and region. Resolvers should use this to indicate that -// a client should fallback and attempt to use it's default resolver to resolve the endpoint. +// EndpointNotFoundError is a sentinel error to indicate that the +// EndpointResolver implementation was unable to resolve an endpoint for the +// given service and region. Resolvers should use this to indicate that an API +// client should fallback and attempt to use it's internal default resolver to +// resolve the endpoint. type EndpointNotFoundError struct { Err error } @@ -41,10 +75,12 @@ func (e *EndpointNotFoundError) Unwrap() error { return e.Err } -// EndpointResolver is an endpoint resolver that can be used to provide or override an endpoint for the given -// service and region. Clients will attempt to use the EndpointResolver first to resolve an endpoint if available. -// If the EndpointResolver returns an EndpointNotFoundError error, clients will fallback to attempting to resolve the endpoint -// using their default endpoint resolver. +// EndpointResolver is an endpoint resolver that can be used to provide or +// override an endpoint for the given service and region. API clients will +// attempt to use the EndpointResolver first to resolve an endpoint if +// available. If the EndpointResolver returns an EndpointNotFoundError error, +// API clients will fallback to attempting to resolve the endpoint using its +// internal default endpoint resolver. type EndpointResolver interface { ResolveEndpoint(service, region string) (Endpoint, error) } @@ -52,7 +88,7 @@ type EndpointResolver interface { // EndpointResolverFunc wraps a function to satisfy the EndpointResolver interface. type EndpointResolverFunc func(service, region string) (Endpoint, error) -// ResolveEndpoint calls the wrapped function and returns the results +// ResolveEndpoint calls the wrapped function and returns the results. func (e EndpointResolverFunc) ResolveEndpoint(service, region string) (Endpoint, error) { return e(service, region) } diff --git a/aws/errors.go b/aws/errors.go index 0a92aec2cd9..f390a08f9ff 100644 --- a/aws/errors.go +++ b/aws/errors.go @@ -1,15 +1,9 @@ package aws -// MissingRegionError is an error that is returned if region configuration is not found. +// MissingRegionError is an error that is returned if region configuration +// value was not found. type MissingRegionError struct{} func (*MissingRegionError) Error() string { - return "could not find region configuration" -} - -// MissingEndpointError is an error that is returned if an endpoint cannot be resolved for a service. -type MissingEndpointError struct{} - -func (*MissingEndpointError) Error() string { - return "'Endpoint' configuration is required for this service" + return "an AWS region is required, but was not found" } diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java index ffb1ed327c0..0b83c20db6c 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java @@ -48,7 +48,7 @@ final class EndpointGenerator implements Runnable { public static final String ADD_MIDDLEWARE_HELPER_NAME = String.format("add%sMiddleware", MIDDLEWARE_NAME); public static final String RESOLVER_INTERFACE_NAME = "EndpointResolver"; public static final String RESOLVER_FUNC_NAME = "EndpointResolverFunc"; - public static final String RESOLVER_OPTIONS = "ResolverOptions"; + public static final String RESOLVER_OPTIONS = "EndpointResolverOptions"; public static final String CLIENT_CONFIG_RESOLVER = "resolveDefaultEndpointConfiguration"; public static final String RESOLVER_CONSTRUCTOR_NAME = "NewDefaultEndpointResolver"; public static final String AWS_ENDPOINT_RESOLVER_HELPER = "WithEndpointResolver"; @@ -217,6 +217,7 @@ private void generateMiddlewareResolverBody(GoStackStepMiddlewareGenerator g, Go w.addUseImports(SmithyGoDependency.FMT); w.addUseImports(SmithyGoDependency.NET_URL); w.addUseImports(AwsGoDependency.AWS_MIDDLEWARE); + w.addUseImports(SmithyGoDependency.SMITHY_MIDDLEWARE); w.addUseImports(SmithyGoDependency.SMITHY_HTTP_TRANSPORT); w.write("req, ok := in.Request.(*smithyhttp.Request)"); @@ -224,22 +225,26 @@ private void generateMiddlewareResolverBody(GoStackStepMiddlewareGenerator g, Go w.write("return out, metadata, fmt.Errorf(\"unknown transport type %T\", in.Request)"); }); w.write(""); + w.openBlock("if m.Resolver == nil {", "}", () -> { w.write("return out, metadata, fmt.Errorf(\"expected endpoint resolver to not be nil\")"); }); w.write(""); + w.write("var endpoint $T", SymbolUtils.createValueSymbolBuilder("Endpoint", AwsGoDependency.AWS_CORE) .build()); w.write("endpoint, err = m.Resolver.ResolveEndpoint(awsmiddleware.GetRegion(ctx), m.Options)"); w.openBlock("if err != nil {", "}", () -> { - w.write("return out, metadata, fmt.Errorf(\"failed to resolve service endpoint\")"); + w.write("return out, metadata, fmt.Errorf(\"failed to resolve service endpoint, %w\", err)"); }); w.write(""); + w.write("req.URL, err = url.Parse(endpoint.URL)"); w.openBlock("if err != nil {", "}", () -> { w.write("return out, metadata, fmt.Errorf(\"failed to parse endpoint URL: %w\", err)"); }); w.write(""); + w.openBlock("if len(awsmiddleware.GetSigningName(ctx)) == 0 {", "}", () -> { w.write("signingName := endpoint.SigningName"); w.openBlock("if len(signingName) == 0 {", "}", () -> { @@ -247,8 +252,12 @@ private void generateMiddlewareResolverBody(GoStackStepMiddlewareGenerator g, Go }); w.write("ctx = awsmiddleware.SetSigningName(ctx, signingName)"); }); + w.write(""); + w.write("ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion)"); + w.write("ctx = smithyhttp.SetHostnameImmutable(ctx, endpoint.HostnameImmutable)"); w.write(""); + w.write("return next.HandleSerialize(ctx, in)"); } @@ -387,6 +396,12 @@ private void generateInternalResolverImplementation(GoWriter writer) { writer.write(""); writer.writeDocs("ResolveEndpoint resolves the service endpoint for the given region and options"); writeInternalResolveEndpointImplementation(writer, resolverImplSymbol, "r", () -> { + // Currently all APIs require a region to derive the endpoint for that API. If there are ever a truly + // region-less API then this should be gated at codegen. + writer.addUseImports(AwsGoDependency.AWS_CORE); + writer.write("if len(region) == 0 { return endpoint, &aws.MissingRegionError{} }"); + writer.write(""); + Symbol sharedOptions = SymbolUtils.createPointableSymbolBuilder("Options", AwsGoDependency.AWS_ENDPOINTS).build(); writer.openBlock("opt := $T{", "}", sharedOptions, () -> { diff --git a/codegen/smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen/customization/presign_url.go.template b/codegen/smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen/customization/presign_url.go.template index 62871f7031c..ad09e456235 100644 --- a/codegen/smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen/customization/presign_url.go.template +++ b/codegen/smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen/customization/presign_url.go.template @@ -55,7 +55,7 @@ func TestClient${operation}_presignURLCustomization(t *testing.T) { return smithyhttp.NopClient{}.Do(r) }), EndpointResolver: EndpointResolverFunc( - func(region string, options ResolverOptions) (aws.Endpoint, error) { + func(region string, options EndpointResolverOptions) (aws.Endpoint, error) { return aws.Endpoint{ URL: "https://service." + region + ".amazonaws.com", SigningRegion: c.ClientRegion, diff --git a/feature/s3/manager/bucket_region_test.go b/feature/s3/manager/bucket_region_test.go index dd56d7d1a69..c0ed0151946 100644 --- a/feature/s3/manager/bucket_region_test.go +++ b/feature/s3/manager/bucket_region_test.go @@ -67,7 +67,7 @@ func TestGetBucketRegion_Exists(t *testing.T) { server := testSetupGetBucketRegionServer(c.RespRegion, c.StatusCode, true) client := s3.New(s3.Options{ - EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.ResolverOptions) (aws.Endpoint, error) { + EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { return aws.Endpoint{ URL: server.URL, }, nil @@ -95,7 +95,7 @@ func TestGetBucketRegion_NotExists(t *testing.T) { defer server.Close() client := s3.New(s3.Options{ - EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.ResolverOptions) (aws.Endpoint, error) { + EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { return aws.Endpoint{ URL: server.URL, }, nil diff --git a/feature/s3/manager/download_test.go b/feature/s3/manager/download_test.go index 7aa2de1ce33..a0af81cf59d 100644 --- a/feature/s3/manager/download_test.go +++ b/feature/s3/manager/download_test.go @@ -464,7 +464,9 @@ func TestDownloadPartBodyRetry_FailRetry(t *testing.T) { } func TestDownloadWithContextCanceled(t *testing.T) { - d := manager.NewDownloader(s3.New(s3.Options{})) + d := manager.NewDownloader(s3.New(s3.Options{ + Region: "mock-region", + })) params := s3.GetObjectInput{ Bucket: aws.String("bucket"), diff --git a/feature/s3/manager/internal/testing/endpoints.go b/feature/s3/manager/internal/testing/endpoints.go index aa2f62ed72c..7f6edf2351d 100644 --- a/feature/s3/manager/internal/testing/endpoints.go +++ b/feature/s3/manager/internal/testing/endpoints.go @@ -6,9 +6,9 @@ import ( ) // EndpointResolverFunc is a mock s3 endpoint resolver that wraps the given function -type EndpointResolverFunc func(region string, options s3.ResolverOptions) (aws.Endpoint, error) +type EndpointResolverFunc func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) // ResolveEndpoint returns the results from the wrapped function. -func (m EndpointResolverFunc) ResolveEndpoint(region string, options s3.ResolverOptions) (aws.Endpoint, error) { +func (m EndpointResolverFunc) ResolveEndpoint(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { return m(region, options) } diff --git a/feature/s3/manager/upload_test.go b/feature/s3/manager/upload_test.go index 17a42c5336d..d42bfa579f3 100644 --- a/feature/s3/manager/upload_test.go +++ b/feature/s3/manager/upload_test.go @@ -741,6 +741,7 @@ func TestSSE(t *testing.T) { func TestUploadWithContextCanceled(t *testing.T) { u := manager.NewUploader(s3.New(s3.Options{ UsePathStyle: true, + Region: "mock-region", })) params := s3.PutObjectInput{ @@ -878,7 +879,7 @@ func TestUploadRetry(t *testing.T) { defer server.Close() client := s3.New(s3.Options{ - EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.ResolverOptions) (aws.Endpoint, error) { + EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { return aws.Endpoint{ URL: server.URL, }, nil diff --git a/service/internal/s3shared/update_endpoint.go b/service/internal/s3shared/update_endpoint.go index 19ab11e248f..46852c4ec80 100644 --- a/service/internal/s3shared/update_endpoint.go +++ b/service/internal/s3shared/update_endpoint.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/awslabs/smithy-go/middleware" - "github.com/awslabs/smithy-go/transport/http" + smithyhttp "github.com/awslabs/smithy-go/transport/http" ) // EnableDualstackMiddleware represents middleware struct for enabling dualstack support @@ -28,7 +28,11 @@ func (u *EnableDualstackMiddleware) HandleSerialize( ) ( out middleware.SerializeOutput, metadata middleware.Metadata, err error, ) { - req, ok := in.Request.(*http.Request) + if smithyhttp.GetHostnameImmutable(ctx) { + return next.HandleSerialize(ctx, in) + } + + req, ok := in.Request.(*smithyhttp.Request) if !ok { return out, metadata, fmt.Errorf("unknown request type %T", req) } diff --git a/service/s3/internal/customizations/doc.go b/service/s3/internal/customizations/doc.go index d9589282c3f..5946e759955 100644 --- a/service/s3/internal/customizations/doc.go +++ b/service/s3/internal/customizations/doc.go @@ -14,6 +14,12 @@ Since serializers serialize by default as path style url, we use customization to modify the endpoint url when `UsePathStyle` option on S3Client is unset or false. This flag will be ignored if `UseAccelerate` option is set to true. +If UseAccelerate is not enabled, and the bucket name is not a valid hostname +label, they SDK will fallback to forcing the request to be made as if +UsePathStyle was enabled. This behavior is also used if UseDualStack is enabled. + +https://docs.aws.amazon.com/AmazonS3/latest/dev/dual-stack-endpoints.html#dual-stack-endpoints-description + Transfer acceleration diff --git a/service/s3/internal/customizations/handle_200_error_test.go b/service/s3/internal/customizations/handle_200_error_test.go index fd49c807c4a..503c4618cb5 100644 --- a/service/s3/internal/customizations/handle_200_error_test.go +++ b/service/s3/internal/customizations/handle_200_error_test.go @@ -14,9 +14,9 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" ) -type EndpointResolverFunc func(region string, options s3.ResolverOptions) (aws.Endpoint, error) +type EndpointResolverFunc func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) -func (fn EndpointResolverFunc) ResolveEndpoint(region string, options s3.ResolverOptions) (endpoint aws.Endpoint, err error) { +func (fn EndpointResolverFunc) ResolveEndpoint(region string, options s3.EndpointResolverOptions) (endpoint aws.Endpoint, err error) { return fn(region, options) } @@ -73,7 +73,7 @@ func TestErrorResponseWith200StatusCode(t *testing.T) { Credentials: unit.StubCredentialsProvider{}, Retryer: aws.NoOpRetryer{}, Region: "mock-region", - EndpointResolver: EndpointResolverFunc(func(region string, options s3.ResolverOptions) (e aws.Endpoint, err error) { + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (e aws.Endpoint, err error) { e.URL = server.URL e.SigningRegion = "us-west-2" return e, err diff --git a/service/s3/internal/customizations/update_endpoint.go b/service/s3/internal/customizations/update_endpoint.go index b355b603135..5a9320d067a 100644 --- a/service/s3/internal/customizations/update_endpoint.go +++ b/service/s3/internal/customizations/update_endpoint.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/awslabs/smithy-go/middleware" - "github.com/awslabs/smithy-go/transport/http" + smithyhttp "github.com/awslabs/smithy-go/transport/http" "github.com/aws/aws-sdk-go-v2/service/internal/s3shared" ) @@ -78,7 +78,11 @@ func (u *updateEndpointMiddleware) HandleSerialize( ) ( out middleware.SerializeOutput, metadata middleware.Metadata, err error, ) { - req, ok := in.Request.(*http.Request) + if smithyhttp.GetHostnameImmutable(ctx) { + return next.HandleSerialize(ctx, in) + } + + req, ok := in.Request.(*smithyhttp.Request) if !ok { return out, metadata, fmt.Errorf("unknown request type %T", req) } @@ -109,15 +113,23 @@ func (u *updateEndpointMiddleware) HandleSerialize( return next.HandleSerialize(ctx, in) } -func (u updateEndpointMiddleware) updateEndpointFromConfig(req *http.Request, bucket string) error { +func (u updateEndpointMiddleware) updateEndpointFromConfig(req *smithyhttp.Request, bucket string) error { // do nothing if path style is enforced if u.usePathStyle { return nil } if !hostCompatibleBucketName(req.URL, bucket) { - // bucket name must be valid to put into the host - return fmt.Errorf("bucket name %s is not compatible with S3", bucket) + // bucket name must be valid to put into the host for accelerate operations. + // For non-accelerate operations the bucket name can stay in the path if + // not valid hostname. + var err error + if u.useAccelerate { + err = fmt.Errorf("bucket name %s is not compatible with S3", bucket) + } + + // No-Op if not using accelerate. + return err } // accelerate is only supported if use path style is disabled diff --git a/service/s3/internal/customizations/update_endpoint_test.go b/service/s3/internal/customizations/update_endpoint_test.go index e4d3318aed1..3fbcf8880e8 100644 --- a/service/s3/internal/customizations/update_endpoint_test.go +++ b/service/s3/internal/customizations/update_endpoint_test.go @@ -3,6 +3,7 @@ package customizations_test import ( "context" "fmt" + "strconv" "strings" "testing" @@ -11,7 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints" "github.com/awslabs/smithy-go/middleware" - "github.com/awslabs/smithy-go/transport/http" + smithyhttp "github.com/awslabs/smithy-go/transport/http" "github.com/aws/aws-sdk-go-v2/service/s3" ) @@ -23,123 +24,214 @@ type s3BucketTest struct { } func TestUpdateEndpointBuild(t *testing.T) { - cases := map[string]struct { - tests []s3BucketTest - useAccelerate bool - useDualstack bool - usePathStyle bool - disableHTTPS bool + cases := map[string]map[string]struct { + tests []s3BucketTest + useAccelerate bool + useDualstack bool + usePathStyle bool + disableHTTPS bool + customEndpoint *aws.Endpoint }{ - "PathStyleBucket": { - usePathStyle: true, - tests: []s3BucketTest{ - {"abc", "https://s3.mock-region.amazonaws.com/abc", ""}, - {"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", ""}, - {"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", ""}, - {"a..bc", "https://s3.mock-region.amazonaws.com/a..bc", ""}, + "default endpoint": { + "PathStyleBucket": { + usePathStyle: true, + tests: []s3BucketTest{ + {"abc", "https://s3.mock-region.amazonaws.com/abc", ""}, + {"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", ""}, + {"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", ""}, + {"a..bc", "https://s3.mock-region.amazonaws.com/a..bc", ""}, + }, }, - }, - "VirtualHostStyleBucket": { - tests: []s3BucketTest{ - {"abc", "https://abc.s3.mock-region.amazonaws.com/", ""}, - {"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", ""}, - {"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", ""}, - {"a..bc", "https://s3.mock-region.amazonaws.com/a..bc", ""}, + "VirtualHostStyleBucket": { + tests: []s3BucketTest{ + {"abc", "https://abc.s3.mock-region.amazonaws.com/", ""}, + {"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", ""}, + {"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", ""}, + {"a..bc", "https://s3.mock-region.amazonaws.com/a..bc", ""}, + }, }, - }, - "Accelerate": { - useAccelerate: true, - tests: []s3BucketTest{ - {"abc", "https://abc.s3-accelerate.amazonaws.com/", ""}, - {"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", "not compatible"}, - {"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", "not compatible"}, + "Accelerate": { + useAccelerate: true, + tests: []s3BucketTest{ + {"abc", "https://abc.s3-accelerate.amazonaws.com/", ""}, + {"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", "not compatible"}, + {"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", "not compatible"}, + }, }, - }, - "AccelerateNoSSLTests": { - useAccelerate: true, - disableHTTPS: true, - tests: []s3BucketTest{ - {"abc", "http://abc.s3-accelerate.amazonaws.com/", ""}, - {"a.b.c", "http://a.b.c.s3-accelerate.amazonaws.com/", ""}, - {"a$b$c", "http://s3.mock-region.amazonaws.com/a%24b%24c", "not compatible"}, + "AccelerateNoSSLTests": { + useAccelerate: true, + disableHTTPS: true, + tests: []s3BucketTest{ + {"abc", "http://abc.s3-accelerate.amazonaws.com/", ""}, + {"a.b.c", "http://a.b.c.s3-accelerate.amazonaws.com/", ""}, + {"a$b$c", "http://s3.mock-region.amazonaws.com/a%24b%24c", "not compatible"}, + }, }, - }, - "DualStack": { - useDualstack: true, - tests: []s3BucketTest{ - {"abc", "https://abc.s3.dualstack.mock-region.amazonaws.com/", ""}, - {"a.b.c", "https://s3.dualstack.mock-region.amazonaws.com/a.b.c", "not compatible"}, - {"a$b$c", "https://s3.dualstack.mock-region.amazonaws.com/a%24b%24c", "not compatible"}, + "DualStack": { + useDualstack: true, + tests: []s3BucketTest{ + {"abc", "https://abc.s3.dualstack.mock-region.amazonaws.com/", ""}, + {"a.b.c", "https://s3.dualstack.mock-region.amazonaws.com/a.b.c", ""}, + {"a$b$c", "https://s3.dualstack.mock-region.amazonaws.com/a%24b%24c", ""}, + }, }, - }, - "DualStackWithPathStyle": { - useDualstack: true, - usePathStyle: true, - tests: []s3BucketTest{ - {"abc", "https://s3.dualstack.mock-region.amazonaws.com/abc", ""}, - {"a.b.c", "https://s3.dualstack.mock-region.amazonaws.com/a.b.c", ""}, - {"a$b$c", "https://s3.dualstack.mock-region.amazonaws.com/a%24b%24c", ""}, + "DualStackWithPathStyle": { + useDualstack: true, + usePathStyle: true, + tests: []s3BucketTest{ + {"abc", "https://s3.dualstack.mock-region.amazonaws.com/abc", ""}, + {"a.b.c", "https://s3.dualstack.mock-region.amazonaws.com/a.b.c", ""}, + {"a$b$c", "https://s3.dualstack.mock-region.amazonaws.com/a%24b%24c", ""}, + }, }, - }, - "AccelerateWithDualStack": { - useAccelerate: true, - useDualstack: true, - tests: []s3BucketTest{ - {"abc", "https://abc.s3-accelerate.dualstack.amazonaws.com/", ""}, - {"a.b.c", "https://s3.dualstack.mock-region.amazonaws.com/a.b.c", "not compatible"}, - {"a$b$c", "https://s3.dualstack.mock-region.amazonaws.com/a%24b%24c", "not compatible"}, + "AccelerateWithDualStack": { + useAccelerate: true, + useDualstack: true, + tests: []s3BucketTest{ + {"abc", "https://abc.s3-accelerate.dualstack.amazonaws.com/", ""}, + {"a.b.c", "https://s3.dualstack.mock-region.amazonaws.com/a.b.c", "not compatible"}, + {"a$b$c", "https://s3.dualstack.mock-region.amazonaws.com/a%24b%24c", "not compatible"}, + }, }, }, - } - - for name, c := range cases { - options := s3.Options{ - Credentials: unit.StubCredentialsProvider{}, - Retryer: aws.NoOpRetryer{}, - Region: "mock-region", - EndpointOptions: endpoints.Options{ - DisableHTTPS: c.disableHTTPS, + "immutable endpoint": { + "PathStyleBucket": { + usePathStyle: true, + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "https://example.region.amazonaws.com/abc", ""}, + {"a$b$c", "https://example.region.amazonaws.com/a%24b%24c", ""}, + {"a.b.c", "https://example.region.amazonaws.com/a.b.c", ""}, + {"a..bc", "https://example.region.amazonaws.com/a..bc", ""}, + }, }, + "VirtualHostStyleBucket": { + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "https://example.region.amazonaws.com/abc", ""}, + {"a$b$c", "https://example.region.amazonaws.com/a%24b%24c", ""}, + {"a.b.c", "https://example.region.amazonaws.com/a.b.c", ""}, + {"a..bc", "https://example.region.amazonaws.com/a..bc", ""}, + }, + }, + "Accelerate": { + useAccelerate: true, + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "https://example.region.amazonaws.com/abc", ""}, + {"a$b$c", "https://example.region.amazonaws.com/a%24b%24c", ""}, + {"a.b.c", "https://example.region.amazonaws.com/a.b.c", ""}, + {"a..bc", "https://example.region.amazonaws.com/a..bc", ""}, + }, + }, + "AccelerateNoSSLTests": { + useAccelerate: true, + disableHTTPS: true, + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "https://example.region.amazonaws.com/abc", ""}, + {"a.b.c", "https://example.region.amazonaws.com/a.b.c", ""}, + {"a$b$c", "https://example.region.amazonaws.com/a%24b%24c", ""}, + }, + }, + "DualStack": { + useDualstack: true, + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "https://example.region.amazonaws.com/abc", ""}, + {"a.b.c", "https://example.region.amazonaws.com/a.b.c", ""}, + {"a$b$c", "https://example.region.amazonaws.com/a%24b%24c", ""}, + }, + }, + }, + } - UsePathStyle: c.usePathStyle, - UseAccelerate: c.useAccelerate, - UseDualstack: c.useDualstack, - } - - svc := s3.New(options) - for i, test := range c.tests { - fm := requestRetrieverMiddleware{} - _, err := svc.ListObjects(context.Background(), - &s3.ListObjectsInput{Bucket: &test.bucket}, - func(options *s3.Options) { - options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { - stack.Serialize.Insert(&fm, "OperationSerializer", middleware.Before) - return nil - }) - }, - ) - - if test.err != "" { - if err == nil { - t.Fatalf("test %d: expected error, got none", i) - } - if a, e := err.Error(), test.err; !strings.Contains(a, e) { - t.Fatalf("%s: %d, expect error code to contain %q, got %q", name, i, e, a) - } - } - - req := fm.request - - if e, a := test.url, req.URL.String(); e != a { - t.Fatalf("%s: %d, expect url %s, got %s", name, i, e, a) + for suitName, cs := range cases { + t.Run(suitName, func(t *testing.T) { + for unitName, c := range cs { + t.Run(unitName, func(t *testing.T) { + options := s3.Options{ + Credentials: unit.StubCredentialsProvider{}, + Retryer: aws.NoOpRetryer{}, + Region: "mock-region", + + HTTPClient: smithyhttp.NopClient{}, + + EndpointOptions: endpoints.Options{ + DisableHTTPS: c.disableHTTPS, + }, + + UsePathStyle: c.usePathStyle, + UseAccelerate: c.useAccelerate, + UseDualstack: c.useDualstack, + } + + if c.customEndpoint != nil { + options.EndpointResolver = s3.EndpointResolverFunc( + func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return *c.customEndpoint, nil + }) + } + + svc := s3.New(options) + for i, test := range c.tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + fm := requestRetrieverMiddleware{} + _, err := svc.ListObjects(context.Background(), + &s3.ListObjectsInput{Bucket: &test.bucket}, + func(options *s3.Options) { + options.APIOptions = append(options.APIOptions, + func(stack *middleware.Stack) error { + stack.Serialize.Insert(&fm, + "OperationSerializer", middleware.Before) + return nil + }) + }, + ) + + if test.err != "" { + if err == nil { + t.Fatalf("test %d: expected error, got none", i) + } + if a, e := err.Error(), test.err; !strings.Contains(a, e) { + t.Fatalf("expect error code to contain %q, got %q", e, a) + } + return + } + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + + req := fm.request.Build(context.Background()) + if e, a := test.url, req.URL.String(); e != a { + t.Fatalf("expect url %s, got %s", e, a) + } + }) + } + }) } - } + }) } } type requestRetrieverMiddleware struct { - request *http.Request + request *smithyhttp.Request } func (*requestRetrieverMiddleware) ID() string { return "S3:requestRetrieverMiddleware" } @@ -149,7 +241,7 @@ func (rm *requestRetrieverMiddleware) HandleSerialize( ) ( out middleware.SerializeOutput, metadata middleware.Metadata, err error, ) { - req, ok := in.Request.(*http.Request) + req, ok := in.Request.(*smithyhttp.Request) if !ok { return out, metadata, fmt.Errorf("unknown request type %T", req) } diff --git a/service/s3control/internal/customizations/update_endpoint_test.go b/service/s3control/internal/customizations/update_endpoint_test.go index f5cc03636f0..ba4abef4b4a 100644 --- a/service/s3control/internal/customizations/update_endpoint_test.go +++ b/service/s3control/internal/customizations/update_endpoint_test.go @@ -3,6 +3,7 @@ package customizations_test import ( "context" "fmt" + "strconv" "strings" "testing" @@ -11,7 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/awslabs/smithy-go/middleware" - "github.com/awslabs/smithy-go/transport/http" + smithyhttp "github.com/awslabs/smithy-go/transport/http" ) type s3controlEndpointTest struct { @@ -22,64 +23,125 @@ type s3controlEndpointTest struct { } func TestUpdateEndpointBuild(t *testing.T) { - cases := map[string]struct { - tests []s3controlEndpointTest - useDualstack bool + cases := map[string]map[string]struct { + tests []s3controlEndpointTest + useDualstack bool + customEndpoint *aws.Endpoint }{ - "DualStack": { - useDualstack: true, - tests: []s3controlEndpointTest{ - {"abc", "123456789012", "https://s3-control.dualstack.mock-region.amazonaws.com/v20180820/bucket/abc", ""}, - {"a.b.c", "123456789012", "https://s3-control.dualstack.mock-region.amazonaws.com/v20180820/bucket/a.b.c", ""}, - {"a$b$c", "123456789012", "https://s3-control.dualstack.mock-region.amazonaws.com/v20180820/bucket/a%24b%24c", ""}, + "default endpoint": { + "default": { + tests: []s3controlEndpointTest{ + {"abc", "123456789012", "https://123456789012.s3-control.mock-region.amazonaws.com/v20180820/bucket/abc", ""}, + {"a.b.c", "123456789012", "https://123456789012.s3-control.mock-region.amazonaws.com/v20180820/bucket/a.b.c", ""}, + {"a$b$c", "123456789012", "https://123456789012.s3-control.mock-region.amazonaws.com/v20180820/bucket/a%24b%24c", ""}, + }, + }, + "DualStack": { + useDualstack: true, + tests: []s3controlEndpointTest{ + {"abc", "123456789012", "https://123456789012.s3-control.dualstack.mock-region.amazonaws.com/v20180820/bucket/abc", ""}, + {"a.b.c", "123456789012", "https://123456789012.s3-control.dualstack.mock-region.amazonaws.com/v20180820/bucket/a.b.c", ""}, + {"a$b$c", "123456789012", "https://123456789012.s3-control.dualstack.mock-region.amazonaws.com/v20180820/bucket/a%24b%24c", ""}, + }, }, }, - } - for name, c := range cases { - options := s3control.Options{ - Credentials: unit.StubCredentialsProvider{}, - Retryer: aws.NoOpRetryer{}, - Region: "mock-region", - UseDualstack: c.useDualstack, - } - - svc := s3control.New(options) - for i, test := range c.tests { - fm := requestRetrieverMiddleware{} - _, err := svc.DeleteBucket(context.Background(), - &s3control.DeleteBucketInput{ - Bucket: &test.bucket, - AccountId: &test.accountID, + "immutable endpoint": { + "default": { + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, }, - func(options *s3control.Options) { - options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { - stack.Serialize.Insert(&fm, "OperationSerializer", middleware.Before) - return nil - }) + tests: []s3controlEndpointTest{ + {"abc", "123456789012", "https://example.region.amazonaws.com/v20180820/bucket/abc", ""}, + {"a.b.c", "123456789012", "https://example.region.amazonaws.com/v20180820/bucket/a.b.c", ""}, + {"a$b$c", "123456789012", "https://example.region.amazonaws.com/v20180820/bucket/a%24b%24c", ""}, }, - ) - - if test.err != "" { - if err == nil { - t.Fatalf("test %d: expected error, got none", i) - } - if a, e := err.Error(), test.err; !strings.Contains(a, e) { - t.Fatalf("%s: %d, expect error code to contain %q, got %q", name, i, e, a) - } - } + }, + "DualStack": { + useDualstack: true, + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3controlEndpointTest{ + {"abc", "123456789012", "https://example.region.amazonaws.com/v20180820/bucket/abc", ""}, + {"a.b.c", "123456789012", "https://example.region.amazonaws.com/v20180820/bucket/a.b.c", ""}, + {"a$b$c", "123456789012", "https://example.region.amazonaws.com/v20180820/bucket/a%24b%24c", ""}, + }, + }, + }, + } + + for suitName, cs := range cases { + t.Run(suitName, func(t *testing.T) { + for unitName, c := range cs { + t.Run(unitName, func(t *testing.T) { + + options := s3control.Options{ + Credentials: unit.StubCredentialsProvider{}, + Retryer: aws.NoOpRetryer{}, + Region: "mock-region", + + HTTPClient: smithyhttp.NopClient{}, + + UseDualstack: c.useDualstack, + } + + if c.customEndpoint != nil { + options.EndpointResolver = s3control.EndpointResolverFunc( + func(region string, options s3control.EndpointResolverOptions) (aws.Endpoint, error) { + return *c.customEndpoint, nil + }) + } + + svc := s3control.New(options) + for i, test := range c.tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + fm := requestRetrieverMiddleware{} + _, err := svc.DeleteBucket(context.Background(), + &s3control.DeleteBucketInput{ + Bucket: &test.bucket, + AccountId: &test.accountID, + }, + func(options *s3control.Options) { + options.APIOptions = append(options.APIOptions, + func(stack *middleware.Stack) error { + stack.Serialize.Insert(&fm, + "OperationSerializer", middleware.Before) + return nil + }) + + }, + ) - req := fm.request + if test.err != "" { + if err == nil { + t.Fatalf("test %d: expected error, got none", i) + } + if a, e := err.Error(), test.err; !strings.Contains(a, e) { + t.Fatalf("expect error code to contain %q, got %q", e, a) + } + return + } + if err != nil { + t.Fatalf("expect no error, got %v", err) + } - if e, a := test.url, req.URL.String(); e != a { - t.Fatalf("%s: %d, expect url %s, got %s", name, i, e, a) + req := fm.request.Build(context.Background()) + if e, a := test.url, req.URL.String(); e != a { + t.Fatalf("expect URL %s, got %s", e, a) + } + }) + } + }) } - } + }) } } type requestRetrieverMiddleware struct { - request *http.Request + request *smithyhttp.Request } func (*requestRetrieverMiddleware) ID() string { return "S3:requestRetrieverMiddleware" } @@ -89,7 +151,7 @@ func (rm *requestRetrieverMiddleware) HandleSerialize( ) ( out middleware.SerializeOutput, metadata middleware.Metadata, err error, ) { - req, ok := in.Request.(*http.Request) + req, ok := in.Request.(*smithyhttp.Request) if !ok { return out, metadata, fmt.Errorf("unknown request type %T", req) }