Skip to content

Commit

Permalink
Merge pull request #362 from hashicorp/b-cut-expired-creds-retry
Browse files Browse the repository at this point in the history 
Avoid retries on expired credentials
  • Loading branch information
@YakDriver
YakDriver committed Feb 22, 2023
2 parents 87c729d + c4d0ab0 commit 1937c18
Show file tree
Hide file tree
Showing 6 changed files with 432 additions and 4 deletions.
2 changes: 1 addition & 1 deletion aws_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func GetAwsConfig(ctx context.Context, c *Config) (context.Context, aws.Config,

if !c.SkipCredsValidation {
if _, _, err := getAccountIDAndPartitionFromSTSGetCallerIdentity(baseCtx, stsClient(baseCtx, awsConfig, c)); err != nil {
return ctx, awsConfig, fmt.Errorf("error validating provider credentials: %w", err)
return ctx, awsConfig, fmt.Errorf("validating provider credentials: %w", err)
}
}

Expand Down
117 changes: 117 additions & 0 deletions aws_config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/aws/smithy-go"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/google/go-cmp/cmp"
Expand Down Expand Up @@ -94,6 +95,52 @@ func TestGetAwsConfig(t *testing.T) {
servicemocks.MockStsGetCallerIdentityValidEndpoint,
},
},
{
Config: &Config{
AccessKey: servicemocks.MockStaticAccessKey,
Region: "us-east-1",
SecretKey: servicemocks.MockStaticSecretKey,
MaxRetries: 100,
},
Description: "ExpiredToken",
ExpectedRegion: "us-east-1",
ExpectedError: func(err error) bool {
return strings.Contains(err.Error(), "ExpiredToken")
},
MockStsEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredToken,
},
},
{
Config: &Config{
AccessKey: servicemocks.MockStaticAccessKey,
Region: "us-east-1",
SecretKey: servicemocks.MockStaticSecretKey,
},
Description: "ExpiredTokenException",
ExpectedRegion: "us-east-1",
ExpectedError: func(err error) bool {
return strings.Contains(err.Error(), "ExpiredTokenException")
},
MockStsEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredTokenException,
},
},
{
Config: &Config{
AccessKey: servicemocks.MockStaticAccessKey,
Region: "us-east-1",
SecretKey: servicemocks.MockStaticSecretKey,
},
Description: "RequestExpired",
ExpectedRegion: "us-east-1",
ExpectedError: func(err error) bool {
return strings.Contains(err.Error(), "RequestExpired")
},
MockStsEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityInvalidBodyRequestExpired,
},
},
{
Config: &Config{
AccessKey: servicemocks.MockStaticAccessKey,
Expand Down Expand Up @@ -3043,6 +3090,76 @@ func TestRetryHandlers(t *testing.T) {
return results
}(),
},
"no retries for ExpiredToken": {
NextHandler: func() middleware.FinalizeHandler {
num := 0
reqsErrs := make([]error, 2)
for i := 0; i < 2; i++ {
reqsErrs[i] = &smithy.OperationError{
ServiceID: "STS",
OperationName: "GetCallerIdentity",
Err: &smithyhttp.ResponseError{
Response: &smithyhttp.Response{
Response: &http.Response{
StatusCode: 403,
},
},
Err: &smithy.GenericAPIError{
Code: "ExpiredToken",
Message: "The security token included in the request is expired",
},
},
}
}
return middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
if num >= len(reqsErrs) {
err = fmt.Errorf("more requests than expected")
} else {
err = reqsErrs[num]
num++
}
return out, metadata, err
})
},
Err: &smithy.OperationError{
ServiceID: "STS",
OperationName: "GetCallerIdentity",
Err: &smithyhttp.ResponseError{
Response: &smithyhttp.Response{
Response: &http.Response{
StatusCode: 403,
},
},
Err: &smithy.GenericAPIError{
Code: "ExpiredToken",
Message: "The security token included in the request is expired",
},
},
},
ExpectResults: func() retry.AttemptResults {
results := retry.AttemptResults{
Results: make([]retry.AttemptResult, 1),
}
results.Results[0] = retry.AttemptResult{
Err: &smithy.OperationError{
ServiceID: "STS",
OperationName: "GetCallerIdentity",
Err: &smithyhttp.ResponseError{
Response: &smithyhttp.Response{
Response: &http.Response{
StatusCode: 403,
},
},
Err: &smithy.GenericAPIError{
Code: "ExpiredToken",
Message: "The security token included in the request is expired",
},
},
},
}
return results
}(),
},
"stops at maxRetries for other network errors": {
NextHandler: func() middleware.FinalizeHandler {
num := 0
Expand Down
42 changes: 42 additions & 0 deletions awsauth_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,48 @@ func TestGetAccountIDAndPartitionFromSTSGetCallerIdentity(t *testing.T) {
},
ErrCount: 1,
},
{
Description: "sts:GetCallerIdentity ExpiredToken with invalid JSON response",
MockEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredToken,
},
ErrCount: 1,
},
{
Description: "sts:GetCallerIdentity ExpiredToken with valid JSON response",
MockEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityValidBodyExpiredToken,
},
ErrCount: 1,
},
{
Description: "sts:GetCallerIdentity ExpiredTokenException with invalid JSON response",
MockEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredTokenException,
},
ErrCount: 1,
},
{
Description: "sts:GetCallerIdentity ExpiredTokenException with valid JSON response",
MockEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityValidBodyExpiredTokenException,
},
ErrCount: 1,
},
{
Description: "sts:GetCallerIdentity RequestExpired with invalid JSON response",
MockEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityInvalidBodyRequestExpired,
},
ErrCount: 1,
},
{
Description: "sts:GetCallerIdentity RequestExpired with valid JSON response",
MockEndpoints: []*servicemocks.MockEndpoint{
servicemocks.MockStsGetCallerIdentityValidBodyRequestExpired,
},
ErrCount: 1,
},
{
Description: "sts:GetCallerIdentity success",
MockEndpoints: []*servicemocks.MockEndpoint{
Expand Down
156 changes: 156 additions & 0 deletions servicemocks/mock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,72 @@ const (
<Message>User: arn:aws:iam::123456789012:user/Bob is not authorized to perform: sts:GetCallerIdentity</Message>
</Error>
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
</ErrorResponse>`
// MockStsGetCallerIdentityValidResponseBodyExpiredToken uses code "ExpiredToken", seemingly the most common
// code. Errors usually have an invalid body but this may be fixed at some point.
MockStsGetCallerIdentityValidResponseBodyExpiredToken = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<Error>
<Type>Sender</Type>
<Code>ExpiredToken</Code>
<Message>The security token included in the request is expired</Message>
</Error>
<ResponseMetadata>
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
</ResponseMetadata>
</ErrorResponse>`
// MockStsGetCallerIdentityInvalidResponseBodyExpiredToken uses code "ExpiredToken", seemingly the most common
// code. Errors usually have an invalid body.
MockStsGetCallerIdentityInvalidResponseBodyExpiredToken = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<Error>
<Type>Sender</Type>
<Code>ExpiredToken</Code>
<Message>The security token included in the request is expired</Message>
</Error>
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
</ErrorResponse>`
// MockStsGetCallerIdentityValidResponseBodyExpiredTokenException uses code "ExpiredTokenException", a more rare code
// but used at least by Fargate. Errors usually have an invalid body but this may change.
MockStsGetCallerIdentityValidResponseBodyExpiredTokenException = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<Error>
<Type>Sender</Type>
<Code>ExpiredTokenException</Code>
<Message>The security token included in the request is expired</Message>
</Error>
<ResponseMetadata>
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
</ResponseMetadata>
</ErrorResponse>`
// MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException uses code "ExpiredTokenException", a more rare code
// but used at least by Fargate. Errors usually have an invalid body but this may change.
MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<Error>
<Type>Sender</Type>
<Code>ExpiredTokenException</Code>
<Message>The security token included in the request is expired</Message>
</Error>
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
</ErrorResponse>`
// MockStsGetCallerIdentityValidResponseBodyRequestExpired uses code "RequestExpired", a code only used in EC2.
// Errors usually have an invalid body but this may change.
MockStsGetCallerIdentityValidResponseBodyRequestExpired = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<Error>
<Type>Sender</Type>
<Code>RequestExpired</Code>
<Message>The security token included in the request is expired</Message>
</Error>
<ResponseMetadata>
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
</ResponseMetadata>
</ErrorResponse>`
// MockStsGetCallerIdentityInvalidResponseBodyRequestExpired uses code "RequestExpired", a code only used in EC2.
// Errors usually have an invalid body but this may change.
MockStsGetCallerIdentityInvalidResponseBodyRequestExpired = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<Error>
<Type>Sender</Type>
<Code>RequestExpired</Code>
<Message>The security token included in the request is expired</Message>
</Error>
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
</ErrorResponse>`
MockStsGetCallerIdentityPartition = `aws`
MockStsGetCallerIdentityValidResponseBody = `<GetCallerIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
Expand Down Expand Up @@ -211,6 +277,96 @@ var (
StatusCode: http.StatusForbidden,
},
}
MockStsGetCallerIdentityInvalidBodyExpiredToken = &MockEndpoint{
Request: &MockRequest{
Body: url.Values{
"Action": []string{"GetCallerIdentity"},
"Version": []string{"2011-06-15"},
}.Encode(),
Method: http.MethodPost,
Uri: "/",
},
Response: &MockResponse{
Body: MockStsGetCallerIdentityInvalidResponseBodyExpiredToken,
ContentType: "text/xml",
StatusCode: http.StatusForbidden,
},
}
MockStsGetCallerIdentityValidBodyExpiredToken = &MockEndpoint{
Request: &MockRequest{
Body: url.Values{
"Action": []string{"GetCallerIdentity"},
"Version": []string{"2011-06-15"},
}.Encode(),
Method: http.MethodPost,
Uri: "/",
},
Response: &MockResponse{
Body: MockStsGetCallerIdentityValidResponseBodyExpiredToken,
ContentType: "text/xml",
StatusCode: http.StatusForbidden,
},
}
MockStsGetCallerIdentityInvalidBodyExpiredTokenException = &MockEndpoint{
Request: &MockRequest{
Body: url.Values{
"Action": []string{"GetCallerIdentity"},
"Version": []string{"2011-06-15"},
}.Encode(),
Method: http.MethodPost,
Uri: "/",
},
Response: &MockResponse{
Body: MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException,
ContentType: "text/xml",
StatusCode: http.StatusForbidden,
},
}
MockStsGetCallerIdentityValidBodyExpiredTokenException = &MockEndpoint{
Request: &MockRequest{
Body: url.Values{
"Action": []string{"GetCallerIdentity"},
"Version": []string{"2011-06-15"},
}.Encode(),
Method: http.MethodPost,
Uri: "/",
},
Response: &MockResponse{
Body: MockStsGetCallerIdentityValidResponseBodyExpiredTokenException,
ContentType: "text/xml",
StatusCode: http.StatusForbidden,
},
}
MockStsGetCallerIdentityInvalidBodyRequestExpired = &MockEndpoint{
Request: &MockRequest{
Body: url.Values{
"Action": []string{"GetCallerIdentity"},
"Version": []string{"2011-06-15"},
}.Encode(),
Method: http.MethodPost,
Uri: "/",
},
Response: &MockResponse{
Body: MockStsGetCallerIdentityInvalidResponseBodyRequestExpired,
ContentType: "text/xml",
StatusCode: http.StatusForbidden,
},
}
MockStsGetCallerIdentityValidBodyRequestExpired = &MockEndpoint{
Request: &MockRequest{
Body: url.Values{
"Action": []string{"GetCallerIdentity"},
"Version": []string{"2011-06-15"},
}.Encode(),
Method: http.MethodPost,
Uri: "/",
},
Response: &MockResponse{
Body: MockStsGetCallerIdentityValidResponseBodyRequestExpired,
ContentType: "text/xml",
StatusCode: http.StatusForbidden,
},
}
MockStsGetCallerIdentityValidEndpoint = &MockEndpoint{
Request: &MockRequest{
Body: url.Values{
Expand Down
8 changes: 8 additions & 0 deletions v2/awsv1shim/session.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,17 @@ func GetSession(ctx context.Context, awsC *awsv2.Config, c *awsbase.Config) (*se
sess.Handlers.Retry.PushBack(func(r *request.Request) {
logger := logging.RetrieveLogger(r.Context())

if r.IsErrorExpired() {
logger.Warn(ctx, "Disabling retries after next request due to expired credentials", map[string]any{
"error": r.Error,
})
r.Retryable = aws.Bool(false)
}

if r.RetryCount < constants.MaxNetworkRetryCount {
return
}

// RequestError: send request failed
// caused by: Post https://FQDN/: dial tcp: lookup FQDN: no such host
if tfawserr.ErrMessageAndOrigErrContain(r.Error, request.ErrCodeRequestError, "send request failed", "no such host") {
Expand Down
Loading

0 comments on commit 1937c18

Please sign in to comment.