From d122ecdfb268e1dba1116f565f2eb2868b70ed07 Mon Sep 17 00:00:00 2001 From: Gerrit Renker Date: Thu, 11 May 2023 14:58:19 -0400 Subject: [PATCH] [aws-cpp-sdk-s3-crt]: use CommonRunTime to support CopyObject Provide `CopyObject` support based on https://github.com/awslabs/aws-c-s3/pull/284 In contrast to the existing `CopyObject` function, objects larger than 5GB are supported, and multipart copies are automatically handled in parallel. --- .../include/aws/s3-crt/S3CrtClient.h | 7 +- .../aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp | 87 ++++++++++++++++--- 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClient.h b/generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClient.h index 7f19451d2f5..cfbae9dc043 100644 --- a/generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClient.h +++ b/generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClient.h @@ -403,11 +403,7 @@ namespace Aws /** * An Async wrapper for CopyObject that queues the request into a thread executor and triggers associated callback when operation has finished. */ - template - void CopyObjectAsync(const CopyObjectRequestT& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr& context = nullptr) const - { - return SubmitAsync(&S3CrtClient::CopyObject, request, handler, context); - } + virtual void CopyObjectAsync(const Model::CopyObjectRequest& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr& context = nullptr) const; /** *

Creates a new S3 bucket. To create a bucket, you must register with Amazon S3 @@ -5587,6 +5583,7 @@ namespace Aws const S3CrtClient *s3CrtClient; GetObjectResponseReceivedHandler getResponseHandler; PutObjectResponseReceivedHandler putResponseHandler; + CopyObjectResponseReceivedHandler copyResponseHandler; std::shared_ptr asyncCallerContext; const Aws::AmazonWebServiceRequest *originalRequest; std::shared_ptr request; diff --git a/generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp b/generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp index 315281714b2..808c0ba99d1 100644 --- a/generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp +++ b/generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp @@ -390,13 +390,13 @@ static void S3CrtRequestFinishCallback(struct aws_s3_meta_request *meta_request, { AWS_UNREFERENCED_PARAM(meta_request); auto *userData = static_cast(user_data); - + if (meta_request_result->error_code != AWS_ERROR_SUCCESS && meta_request_result->response_status == 0) { /* client side error */ userData->response->SetClientErrorType(CoreErrors::NETWORK_CONNECTION); Aws::StringStream ss; - ss << "crtCode: " << meta_request_result->error_code - << ", " << aws_error_name(meta_request_result->error_code) + ss << "crtCode: " << meta_request_result->error_code + << ", " << aws_error_name(meta_request_result->error_code) << " - " << aws_error_str(meta_request_result->error_code); userData->response->SetClientErrorMessage(ss.str()); userData->response->SetResponseCode(HttpResponseCode::REQUEST_NOT_MADE); @@ -761,29 +761,94 @@ CompleteMultipartUploadOutcome S3CrtClient::CompleteMultipartUpload(const Comple return CompleteMultipartUploadOutcome(MakeRequest(request, endpointResolutionOutcome.GetResult(), Aws::Http::HttpMethod::HTTP_POST)); } -CopyObjectOutcome S3CrtClient::CopyObject(const CopyObjectRequest& request) const +static void CopyObjectRequestShutdownCallback(void *user_data) { - AWS_OPERATION_GUARD(CopyObject); - AWS_OPERATION_CHECK_PTR(m_endpointProvider, CopyObject, CoreErrors, CoreErrors::ENDPOINT_RESOLUTION_FAILURE); + auto *userData = static_cast(user_data); + S3Crt::Model::CopyObjectOutcome outcome(userData->s3CrtClient->GenerateXmlOutcome(userData->response)); + + userData->copyResponseHandler(userData->s3CrtClient, *(reinterpret_cast(userData->originalRequest)), std::move(outcome), userData->asyncCallerContext); + Aws::Delete(userData); +} + +void S3CrtClient::CopyObjectAsync(const Model::CopyObjectRequest& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr& handlerContext) const +{ + if (!m_endpointProvider) { + return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Endpoint provider is not initialized", false)), handlerContext); + } if (!request.BucketHasBeenSet()) { AWS_LOGSTREAM_ERROR("CopyObject", "Required field: Bucket, is not set"); - return CopyObjectOutcome(Aws::Client::AWSError(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Bucket]", false)); + return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Bucket]", false)), handlerContext); } if (!request.CopySourceHasBeenSet()) { AWS_LOGSTREAM_ERROR("CopyObject", "Required field: CopySource, is not set"); - return CopyObjectOutcome(Aws::Client::AWSError(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [CopySource]", false)); + return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [CopySource]", false)), handlerContext); } if (!request.KeyHasBeenSet()) { AWS_LOGSTREAM_ERROR("CopyObject", "Required field: Key, is not set"); - return CopyObjectOutcome(Aws::Client::AWSError(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Key]", false)); + return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Key]", false)), handlerContext); } + ResolveEndpointOutcome endpointResolutionOutcome = m_endpointProvider->ResolveEndpoint(request.GetEndpointContextParams()); - AWS_OPERATION_CHECK_SUCCESS(endpointResolutionOutcome, CopyObject, CoreErrors, CoreErrors::ENDPOINT_RESOLUTION_FAILURE, endpointResolutionOutcome.GetError().GetMessage()); + if (!endpointResolutionOutcome.IsSuccess()) { + handler(this, request, CopyObjectOutcome(Aws::Client::AWSError( + CoreErrors::ENDPOINT_RESOLUTION_FAILURE, "ENDPOINT_RESOLUTION_FAILURE", endpointResolutionOutcome.GetError().GetMessage(), false)), handlerContext); + return; + } endpointResolutionOutcome.GetResult().AddPathSegments(request.GetKey()); - return CopyObjectOutcome(MakeRequest(request, endpointResolutionOutcome.GetResult(), Aws::Http::HttpMethod::HTTP_PUT)); + + CrtRequestCallbackUserData *userData = Aws::New(ALLOCATION_TAG); + aws_s3_meta_request_options options; + AWS_ZERO_STRUCT(options); + aws_uri endpoint; + AWS_ZERO_STRUCT(endpoint); + options.endpoint = &endpoint; + std::unique_ptr endpointCleanup { options.endpoint, &aws_uri_clean_up }; + + userData->copyResponseHandler = handler; + userData->asyncCallerContext = handlerContext; + InitCommonCrtRequestOption(userData, &options, &request, endpointResolutionOutcome.GetResult().GetURI(), Aws::Http::HttpMethod::HTTP_PUT); + options.shutdown_callback = CopyObjectRequestShutdownCallback; + options.type = AWS_S3_META_REQUEST_TYPE_COPY_OBJECT; + + struct aws_signing_config_aws signing_config_override = m_s3CrtSigningConfig; + if (endpointResolutionOutcome.GetResult().GetAttributes() && endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningRegion()) { + signing_config_override.region = Aws::Crt::ByteCursorFromCString(endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningRegion()->c_str()); + } + if (endpointResolutionOutcome.GetResult().GetAttributes() && endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningRegionSet()) { + signing_config_override.region = Aws::Crt::ByteCursorFromCString(endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningRegionSet()->c_str()); + } + if (endpointResolutionOutcome.GetResult().GetAttributes() && endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningName()) { + signing_config_override.service = Aws::Crt::ByteCursorFromCString(endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningName()->c_str()); + } + options.signing_config = &signing_config_override; + + std::shared_ptr crtHttpRequest = userData->request->ToCrtHttpRequest(); + options.message= crtHttpRequest->GetUnderlyingMessage(); + userData->crtHttpRequest = crtHttpRequest; + + if (aws_s3_client_make_meta_request(m_s3CrtClient, &options) == nullptr) + { + return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext); + } +} + +CopyObjectOutcome S3CrtClient::CopyObject(const CopyObjectRequest& request) const +{ + AWS_OPERATION_GUARD(CopyObject); + Aws::Utils::Threading::Semaphore sem(0, 1); + CopyObjectOutcome res; + + auto handler = CopyObjectResponseReceivedHandler{[&](const S3CrtClient*, const CopyObjectRequest&, CopyObjectOutcome outcome, const std::shared_ptr &) { + res = std::move(outcome); + sem.ReleaseAll(); + }}; + + S3CrtClient::CopyObjectAsync(request, handler, nullptr); + sem.WaitOne(); + return res; } CreateBucketOutcome S3CrtClient::CreateBucket(const CreateBucketRequest& request) const