diff --git a/include/aws/s3/private/s3_auto_ranged_get.h b/include/aws/s3/private/s3_auto_ranged_get.h index 4e251c5d1..54bc9143a 100644 --- a/include/aws/s3/private/s3_auto_ranged_get.h +++ b/include/aws/s3/private/s3_auto_ranged_get.h @@ -10,6 +10,7 @@ enum aws_s3_auto_ranged_get_request_type { AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART, + AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE, }; struct aws_s3_auto_ranged_get { @@ -26,6 +27,9 @@ struct aws_s3_auto_ranged_get { size_t total_object_size; + uint32_t get_without_range : 1; + uint32_t get_without_range_sent : 1; + uint32_t get_without_range_completed : 1; } synced_data; }; diff --git a/include/aws/s3/private/s3_request_messages.h b/include/aws/s3/private/s3_request_messages.h index 13df186eb..b33df3a89 100644 --- a/include/aws/s3/private/s3_request_messages.h +++ b/include/aws/s3/private/s3_request_messages.h @@ -7,6 +7,7 @@ */ #include +#include #include struct aws_allocator; @@ -30,7 +31,8 @@ struct aws_http_message *aws_s3_get_object_message_new( struct aws_allocator *allocator, struct aws_http_message *base_message, uint32_t part_number, - size_t part_size); + size_t part_size, + bool has_range); /* Create an HTTP request for an S3 Put Object request, using the original request as a basis. Creates and assigns a * body stream using the passed in buffer. If multipart is not needed, part number and upload_id can be 0 and NULL, diff --git a/source/s3_auto_ranged_get.c b/source/s3_auto_ranged_get.c index 22b1259fd..f092596cc 100644 --- a/source/s3_auto_ranged_get.c +++ b/source/s3_auto_ranged_get.c @@ -17,6 +17,8 @@ #endif const uint32_t s_conservative_max_requests_in_flight = 8; +const struct aws_byte_cursor g_application_xml_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("application/xml"); +const struct aws_byte_cursor g_object_size_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ActualObjectSize"); static void s_s3_meta_request_auto_ranged_get_destroy(struct aws_s3_meta_request *meta_request); @@ -100,6 +102,31 @@ static void s_s3_meta_request_auto_ranged_get_destroy(struct aws_s3_meta_request aws_mem_release(meta_request->allocator, auto_ranged_get); } +/* Check the finish result of meta request, in case of the request failed because of downloading an empty file */ +static bool s_check_empty_file_download_error(struct aws_s3_request *failed_request) { + struct aws_http_headers *failed_headers = failed_request->send_data.response_headers; + struct aws_byte_buf failed_body = failed_request->send_data.response_body; + if (failed_headers && failed_body.capacity > 0) { + struct aws_byte_cursor content_type; + AWS_ZERO_STRUCT(content_type); + if (!aws_http_headers_get(failed_headers, g_content_type_header_name, &content_type)) { + /* Content type found */ + if (aws_byte_cursor_eq_ignore_case(&content_type, &g_application_xml_value)) { + /* XML response */ + struct aws_byte_cursor body_cursor = aws_byte_cursor_from_buf(&failed_body); + struct aws_string *size = + get_top_level_xml_tag_value(failed_request->allocator, &g_object_size_value, &body_cursor); + bool check_size = aws_string_eq_c_str(size, "0"); + aws_string_destroy(size); + if (check_size) { + return true; + } + } + } + } + return false; +} + static bool s_s3_auto_ranged_get_update( struct aws_s3_meta_request *meta_request, uint32_t flags, @@ -161,6 +188,28 @@ static bool s_s3_auto_ranged_get_update( goto has_work_remaining; } + if (auto_ranged_get->synced_data.get_without_range) { + if (auto_ranged_get->synced_data.get_without_range_sent) { + if (auto_ranged_get->synced_data.get_without_range_completed) { + goto no_work_remaining; + } else { + goto has_work_remaining; + } + } + if (out_request == NULL) { + goto has_work_remaining; + } + + request = aws_s3_request_new( + meta_request, + AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE, + 1, + AWS_S3_REQUEST_DESC_RECORD_RESPONSE_HEADERS); + + auto_ranged_get->synced_data.get_without_range_sent = true; + goto has_work_remaining; + } + /* If we have gotten a response for the first request, then the total number of parts for the object is now * known. Continue sending parts until the total number of parts is reached.*/ if (auto_ranged_get->synced_data.num_parts_requested < auto_ranged_get->synced_data.total_num_parts) { @@ -190,6 +239,15 @@ static bool s_s3_auto_ranged_get_update( goto has_work_remaining; } + if (auto_ranged_get->synced_data.get_without_range) { + if (auto_ranged_get->synced_data.get_without_range_sent && + !auto_ranged_get->synced_data.get_without_range_completed) { + goto has_work_remaining; + } else { + goto no_work_remaining; + } + } + /* If some parts are still being delivered to the caller, then wait for those to finish. */ if (meta_request->synced_data.num_parts_delivery_completed < meta_request->synced_data.num_parts_delivery_sent) { @@ -249,11 +307,13 @@ static int s_s3_auto_ranged_get_prepare_request( struct aws_http_message *message = NULL; - AWS_ASSERT(request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART); - /* Generate a new ranged get request based on the original message. */ message = aws_s3_get_object_message_new( - meta_request->allocator, meta_request->initial_request_message, request->part_number, meta_request->part_size); + meta_request->allocator, + meta_request->initial_request_message, + request->part_number, + meta_request->part_size, + request->request_tag != AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE); if (message == NULL) { AWS_LOGF_ERROR( @@ -294,66 +354,67 @@ static void s_s3_auto_ranged_get_request_finished( struct aws_s3_auto_ranged_get *auto_ranged_get = meta_request->impl; - AWS_ASSERT(request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART); - uint32_t num_parts = 0; if (error_code == AWS_ERROR_SUCCESS && request->part_number == 1) { - struct aws_byte_cursor content_range_header_value; - - if (aws_http_headers_get( - request->send_data.response_headers, g_content_range_header_name, &content_range_header_value)) { - AWS_LOGF_ERROR( - AWS_LS_S3_META_REQUEST, - "id=%p Could not find content range header for request %p", - (void *)meta_request, - (void *)request); - - error_code = AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER; - goto error_encountered; - } - - uint64_t range_start = 0; - uint64_t range_end = 0; uint64_t total_object_size = 0; + if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART) { + struct aws_byte_cursor content_range_header_value; + + if (aws_http_headers_get( + request->send_data.response_headers, g_content_range_header_name, &content_range_header_value)) { + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, + "id=%p Could not find content range header for request %p", + (void *)meta_request, + (void *)request); + + error_code = AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER; + goto error_encountered; + } - /* The memory the byte cursor refers to should be valid, but if it's referring to a buffer that was - * previously used, the null terminating character may not be where we expect. We copy to a string to - * ensure that our null terminating character placement corresponds with the length. */ - struct aws_string *content_range_header_value_str = - aws_string_new_from_cursor(meta_request->allocator, &content_range_header_value); - - /* Format of header is: "bytes StartByte-EndByte/TotalObjectSize" */ - sscanf( - (const char *)content_range_header_value_str->bytes, - "bytes %" PRIu64 "-%" PRIu64 "/%" PRIu64, - &range_start, - &range_end, - &total_object_size); - - aws_string_destroy(content_range_header_value_str); - content_range_header_value_str = NULL; - - if (total_object_size == 0) { - AWS_LOGF_ERROR(AWS_LS_S3_META_REQUEST, "id=%p Get Object has invalid content range.", (void *)meta_request); - error_code = AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER; - goto error_encountered; - } + uint64_t range_start = 0; + uint64_t range_end = 0; + + /* The memory the byte cursor refers to should be valid, but if it's referring to a buffer that was + * previously used, the null terminating character may not be where we expect. We copy to a string to + * ensure that our null terminating character placement corresponds with the length. */ + struct aws_string *content_range_header_value_str = + aws_string_new_from_cursor(meta_request->allocator, &content_range_header_value); + + /* Format of header is: "bytes StartByte-EndByte/TotalObjectSize" */ + sscanf( + (const char *)content_range_header_value_str->bytes, + "bytes %" PRIu64 "-%" PRIu64 "/%" PRIu64, + &range_start, + &range_end, + &total_object_size); + + aws_string_destroy(content_range_header_value_str); + content_range_header_value_str = NULL; + + if (total_object_size == 0) { + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, "id=%p Get Object has invalid content range.", (void *)meta_request); + error_code = AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER; + goto error_encountered; + } - num_parts = (uint32_t)(total_object_size / meta_request->part_size); + num_parts = (uint32_t)(total_object_size / meta_request->part_size); - if (total_object_size % meta_request->part_size) { - ++num_parts; - } + if (total_object_size % meta_request->part_size) { + ++num_parts; + } - AWS_LOGF_DEBUG( - AWS_LS_S3_META_REQUEST, - "id=%p Object being requested is %" PRIu64 " bytes which will have %d parts based off of a %" PRIu64 - " part size.", - (void *)meta_request, - total_object_size, - num_parts, - (uint64_t)meta_request->part_size); + AWS_LOGF_DEBUG( + AWS_LS_S3_META_REQUEST, + "id=%p Object being requested is %" PRIu64 " bytes which will have %d parts based off of a %" PRIu64 + " part size.", + (void *)meta_request, + total_object_size, + num_parts, + (uint64_t)meta_request->part_size); + } if (meta_request->headers_callback != NULL) { struct aws_http_headers *response_headers = aws_http_headers_new(meta_request->allocator); @@ -381,28 +442,44 @@ static void s_s3_auto_ranged_get_request_finished( aws_s3_meta_request_lock_synced_data(meta_request); - ++auto_ranged_get->synced_data.num_parts_completed; + if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART) { + ++auto_ranged_get->synced_data.num_parts_completed; - if (error_code == AWS_ERROR_SUCCESS) { - ++auto_ranged_get->synced_data.num_parts_successful; + if (error_code == AWS_ERROR_SUCCESS) { + ++auto_ranged_get->synced_data.num_parts_successful; - if (request->part_number == 1) { - AWS_ASSERT(num_parts > 0); - auto_ranged_get->synced_data.total_num_parts = num_parts; - } + if (request->part_number == 1) { + AWS_ASSERT(num_parts > 0); + auto_ranged_get->synced_data.total_num_parts = num_parts; + } - aws_s3_meta_request_stream_response_body_synced(meta_request, request); + aws_s3_meta_request_stream_response_body_synced(meta_request, request); - AWS_LOGF_DEBUG( - AWS_LS_S3_META_REQUEST, - "id=%p: %d out of %d parts have completed.", - (void *)meta_request, - (auto_ranged_get->synced_data.num_parts_successful + auto_ranged_get->synced_data.num_parts_failed), - auto_ranged_get->synced_data.total_num_parts); - } else { - ++auto_ranged_get->synced_data.num_parts_failed; - - aws_s3_meta_request_set_fail_synced(meta_request, request, error_code); + AWS_LOGF_DEBUG( + AWS_LS_S3_META_REQUEST, + "id=%p: %d out of %d parts have completed.", + (void *)meta_request, + (auto_ranged_get->synced_data.num_parts_successful + auto_ranged_get->synced_data.num_parts_failed), + auto_ranged_get->synced_data.total_num_parts); + } else { + ++auto_ranged_get->synced_data.num_parts_failed; + if (s_check_empty_file_download_error(request)) { + AWS_LOGF_DEBUG( + AWS_LS_S3_META_REQUEST, + "id=%p Getting an empty file, create a new request without range header to fetch the empty " + "file", + (void *)meta_request); + auto_ranged_get->synced_data.get_without_range = true; + } else { + aws_s3_meta_request_set_fail_synced(meta_request, request, error_code); + } + } + } else if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE) { + AWS_LOGF_DEBUG(AWS_LS_S3_META_REQUEST, "id=%p Get empty file completed", (void *)meta_request); + auto_ranged_get->synced_data.get_without_range_completed = true; + if (error_code != AWS_ERROR_SUCCESS) { + aws_s3_meta_request_set_fail_synced(meta_request, request, error_code); + } } aws_s3_meta_request_unlock_synced_data(meta_request); diff --git a/source/s3_request_messages.c b/source/s3_request_messages.c index 67623b2fe..e3a2faa4c 100644 --- a/source/s3_request_messages.c +++ b/source/s3_request_messages.c @@ -33,7 +33,8 @@ struct aws_http_message *aws_s3_get_object_message_new( struct aws_allocator *allocator, struct aws_http_message *base_message, uint32_t part_number, - size_t part_size) { + size_t part_size, + bool has_range) { AWS_PRECONDITION(allocator); AWS_PRECONDITION(base_message); @@ -44,7 +45,7 @@ struct aws_http_message *aws_s3_get_object_message_new( return NULL; } - if (part_number > 0) { + if (part_number > 0 && has_range) { if (s_s3_message_util_add_content_range_header(part_number - 1, part_size, message)) { goto error_clean_up; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a5da3c645..304053daf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -30,6 +30,7 @@ add_net_test_case(test_s3_get_object_tls_disabled) add_net_test_case(test_s3_get_object_tls_enabled) add_net_test_case(test_s3_get_object_tls_default) add_net_test_case(test_s3_get_object_less_than_part_size) +add_net_test_case(test_s3_get_object_empty_object) add_net_test_case(test_s3_get_object_multiple) add_net_test_case(test_s3_get_object_sse_kms) add_net_test_case(test_s3_get_object_sse_aes256) @@ -39,7 +40,9 @@ add_net_test_case(test_s3_put_object_tls_disabled) add_net_test_case(test_s3_put_object_tls_enabled) add_net_test_case(test_s3_put_object_tls_default) add_net_test_case(test_s3_multipart_put_object_with_acl) +add_net_test_case(test_s3_put_object_multiple) add_net_test_case(test_s3_put_object_less_than_part_size) +add_net_test_case(test_s3_put_object_empty_object) add_net_test_case(test_s3_put_object_with_part_remainder) add_net_test_case(test_s3_put_object_sse_kms) add_net_test_case(test_s3_put_object_sse_kms_multipart) diff --git a/tests/s3_data_plane_tests.c b/tests/s3_data_plane_tests.c index 891387464..933e32315 100644 --- a/tests/s3_data_plane_tests.c +++ b/tests/s3_data_plane_tests.c @@ -441,6 +441,14 @@ static int s_test_s3_get_object_multiple(struct aws_allocator *allocator, void * return 0; } +AWS_TEST_CASE(test_s3_get_object_empty_object, s_test_s3_get_object_empty_default) +static int s_test_s3_get_object_empty_default(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + return (s_test_s3_get_object_helper( + allocator, AWS_S3_TLS_ENABLED, 0, aws_byte_cursor_from_c_str("/get_object_test_0MB.txt"))); +} + AWS_TEST_CASE(test_s3_get_object_sse_kms, s_test_s3_get_object_sse_kms) static int s_test_s3_get_object_sse_kms(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -572,6 +580,100 @@ static int s_test_s3_multipart_put_object_with_acl(struct aws_allocator *allocat return 0; } +AWS_TEST_CASE(test_s3_put_object_multiple, s_test_s3_put_object_multiple) +static int s_test_s3_put_object_multiple(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_meta_request *meta_requests[2]; + struct aws_s3_meta_request_test_results meta_request_test_results[2]; + struct aws_http_message *messages[2]; + struct aws_input_stream *input_streams[2]; + struct aws_byte_buf input_stream_buffers[2]; + size_t num_meta_requests = sizeof(meta_requests) / sizeof(struct aws_s3_meta_request *); + + ASSERT_TRUE( + num_meta_requests == (sizeof(meta_request_test_results) / sizeof(struct aws_s3_meta_request_test_results))); + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client_config client_config; + AWS_ZERO_STRUCT(client_config); + + ASSERT_SUCCESS(aws_s3_tester_bind_client( + &tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION | AWS_S3_TESTER_BIND_CLIENT_SIGNING)); + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + + struct aws_string *host_name = + aws_s3_tester_build_endpoint_string(allocator, &g_test_bucket_name, &g_test_s3_region); + + for (size_t i = 0; i < num_meta_requests; ++i) { + AWS_ZERO_STRUCT(meta_request_test_results[i]); + char object_path_buffer[128] = ""; + snprintf(object_path_buffer, sizeof(object_path_buffer), "/get_object_test_10MB_%zu.txt", i); + AWS_ZERO_STRUCT(input_stream_buffers[i]); + aws_s3_create_test_buffer(allocator, 10 * 1024ULL * 1024ULL, &input_stream_buffers[i]); + struct aws_byte_cursor test_body_cursor = aws_byte_cursor_from_buf(&input_stream_buffers[i]); + input_streams[i] = aws_input_stream_new_from_cursor(allocator, &test_body_cursor); + struct aws_byte_cursor test_object_path = aws_byte_cursor_from_c_str(object_path_buffer); + messages[i] = aws_s3_test_put_object_request_new( + allocator, + aws_byte_cursor_from_string(host_name), + test_object_path, + g_test_body_content_type, + input_streams[i], + 0); + struct aws_s3_meta_request_options options; + AWS_ZERO_STRUCT(options); + options.type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT; + options.message = messages[i]; + + ASSERT_SUCCESS(aws_s3_tester_bind_meta_request(&tester, &options, &meta_request_test_results[i])); + + /* Trigger accelerating of our Put Object request. */ + meta_requests[i] = aws_s3_client_make_meta_request(client, &options); + + ASSERT_TRUE(meta_requests[i] != NULL); + } + + /* Wait for the request to finish. */ + aws_s3_tester_wait_for_meta_request_finish(&tester); + + aws_s3_tester_lock_synced_data(&tester); + ASSERT_TRUE(tester.synced_data.finish_error_code == AWS_ERROR_SUCCESS); + aws_s3_tester_unlock_synced_data(&tester); + + for (size_t i = 0; i < num_meta_requests; ++i) { + aws_s3_meta_request_release(meta_requests[i]); + meta_requests[i] = NULL; + } + + aws_s3_tester_wait_for_meta_request_shutdown(&tester); + + for (size_t i = 0; i < num_meta_requests; ++i) { + aws_s3_tester_validate_get_object_results(&meta_request_test_results[i], 0); + aws_s3_meta_request_test_results_clean_up(&meta_request_test_results[i]); + } + + for (size_t i = 0; i < num_meta_requests; ++i) { + aws_http_message_release(messages[i]); + aws_input_stream_destroy(input_streams[i]); + aws_byte_buf_clean_up(&input_stream_buffers[i]); + } + + aws_string_destroy(host_name); + host_name = NULL; + + aws_s3_client_release(client); + client = NULL; + + aws_s3_tester_clean_up(&tester); + + return 0; +} + AWS_TEST_CASE(test_s3_put_object_less_than_part_size, s_test_s3_put_object_less_than_part_size) static int s_test_s3_put_object_less_than_part_size(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -601,6 +703,34 @@ static int s_test_s3_put_object_less_than_part_size(struct aws_allocator *alloca return 0; } +AWS_TEST_CASE(test_s3_put_object_empty_object, s_test_s3_put_object_empty_object) +static int s_test_s3_put_object_empty_object(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client_config client_config; + AWS_ZERO_STRUCT(client_config); + + ASSERT_SUCCESS(aws_s3_tester_bind_client( + &tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION | AWS_S3_TESTER_BIND_CLIENT_SIGNING)); + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + + ASSERT_TRUE(client != NULL); + + ASSERT_SUCCESS(aws_s3_tester_send_put_object_meta_request( + &tester, client, 0, AWS_S3_TESTER_SEND_META_REQUEST_EXPECT_SUCCESS, NULL)); + + aws_s3_client_release(client); + client = NULL; + + aws_s3_tester_clean_up(&tester); + + return 0; +} + AWS_TEST_CASE(test_s3_put_object_sse_kms, s_test_s3_put_object_sse_kms) static int s_test_s3_put_object_sse_kms(struct aws_allocator *allocator, void *ctx) { (void)ctx;