diff --git a/include/aws/s3/private/s3_meta_request_impl.h b/include/aws/s3/private/s3_meta_request_impl.h index e4ae968d..560e71c6 100644 --- a/include/aws/s3/private/s3_meta_request_impl.h +++ b/include/aws/s3/private/s3_meta_request_impl.h @@ -176,11 +176,6 @@ struct aws_s3_meta_request { aws_s3_meta_request_telemetry_fn *telemetry_callback; aws_s3_meta_request_upload_review_fn *upload_review_callback; - /* Customer specified callbacks to be called by our specialized callback to calculate the response checksum. */ - aws_s3_meta_request_headers_callback_fn *headers_user_callback_after_checksum; - aws_s3_meta_request_receive_body_callback_fn *body_user_callback_after_checksum; - aws_s3_meta_request_finish_fn *finish_user_callback_after_checksum; - enum aws_s3_meta_request_type type; struct aws_string *s3express_session_host; diff --git a/include/aws/s3/private/s3_util.h b/include/aws/s3/private/s3_util.h index f9c95f94..19f3e737 100644 --- a/include/aws/s3/private/s3_util.h +++ b/include/aws/s3/private/s3_util.h @@ -36,6 +36,7 @@ struct aws_http_message; struct aws_s3_client; struct aws_s3_request; struct aws_s3_meta_request; +struct aws_s3_checksum; struct aws_cached_signing_config_aws { struct aws_allocator *allocator; @@ -323,6 +324,16 @@ int aws_s3_crt_error_code_from_recoverable_server_error_code_string(struct aws_b AWS_S3_API void aws_s3_request_finish_up_metrics_synced(struct aws_s3_request *request, struct aws_s3_meta_request *meta_request); +/* Check the response headers for checksum to verify, return a running checksum based on the algorithm found. If no + * checksum found from header, return null. */ +AWS_S3_API +int aws_s3_check_headers_for_checksum( + struct aws_s3_meta_request *meta_request, + const struct aws_http_headers *headers, + struct aws_s3_checksum **out_checksum, + struct aws_byte_buf *out_checksum_buffer, + bool meta_request_level); + AWS_EXTERN_C_END #endif /* AWS_S3_UTIL_H */ diff --git a/source/s3_auto_ranged_get.c b/source/s3_auto_ranged_get.c index a71d418e..e68738cc 100644 --- a/source/s3_auto_ranged_get.c +++ b/source/s3_auto_ranged_get.c @@ -809,11 +809,21 @@ static void s_s3_auto_ranged_get_request_finished( error_code = AWS_ERROR_SUCCESS; found_object_size = true; - if (!empty_file_error && meta_request->headers_callback != NULL) { - struct aws_http_headers *response_headers = aws_http_headers_new(meta_request->allocator); - - copy_http_headers(request->send_data.response_headers, response_headers); + /* Check for checksums if requested to */ + if (meta_request->checksum_config.validate_response_checksum) { + if (aws_s3_check_headers_for_checksum( + meta_request, + request->send_data.response_headers, + &meta_request->meta_request_level_running_response_sum, + &meta_request->meta_request_level_response_header_checksum, + true) != AWS_OP_SUCCESS) { + error_code = aws_last_error_or_unknown(); + goto update_synced_data; + } + } + if (!empty_file_error && meta_request->headers_callback != NULL) { + /* Modify the header received to fake the header for the whole meta request. */ if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_GET_OBJECT_WITH_RANGE || request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_GET_OBJECT_WITH_PART_NUMBER_1) { @@ -828,12 +838,12 @@ static void s_s3_auto_ranged_get_request_finished( object_range_end, object_size); aws_http_headers_set( - response_headers, + request->send_data.response_headers, g_content_range_header_name, aws_byte_cursor_from_c_str(content_range_buffer)); } else { /* content range isn't applicable. */ - aws_http_headers_erase(response_headers, g_content_range_header_name); + aws_http_headers_erase(request->send_data.response_headers, g_content_range_header_name); } } @@ -841,19 +851,19 @@ static void s_s3_auto_ranged_get_request_finished( char content_length_buffer[64] = ""; snprintf(content_length_buffer, sizeof(content_length_buffer), "%" PRIu64, content_length); aws_http_headers_set( - response_headers, g_content_length_header_name, aws_byte_cursor_from_c_str(content_length_buffer)); + request->send_data.response_headers, + g_content_length_header_name, + aws_byte_cursor_from_c_str(content_length_buffer)); if (meta_request->headers_callback( meta_request, - response_headers, + request->send_data.response_headers, s_s3_auto_ranged_get_success_status(meta_request), meta_request->user_data)) { error_code = aws_last_error_or_unknown(); } meta_request->headers_callback = NULL; - - aws_http_headers_release(response_headers); } } diff --git a/source/s3_auto_ranged_put.c b/source/s3_auto_ranged_put.c index cb57b9d7..68c90131 100644 --- a/source/s3_auto_ranged_put.c +++ b/source/s3_auto_ranged_put.c @@ -1568,15 +1568,11 @@ static void s_s3_auto_ranged_put_request_finished( case AWS_S3_AUTO_RANGED_PUT_REQUEST_TAG_COMPLETE_MULTIPART_UPLOAD: { if (error_code == AWS_ERROR_SUCCESS && meta_request->headers_callback != NULL) { - struct aws_http_headers *final_response_headers = aws_http_headers_new(meta_request->allocator); - - /* Copy all the response headers from this request. */ - copy_http_headers(request->send_data.response_headers, final_response_headers); - /* Copy over any response headers that we've previously determined are needed for this final * response. */ - copy_http_headers(auto_ranged_put->synced_data.needed_response_headers, final_response_headers); + copy_http_headers( + auto_ranged_put->synced_data.needed_response_headers, request->send_data.response_headers); struct aws_byte_cursor xml_doc = aws_byte_cursor_from_buf(&request->send_data.response_body); @@ -1597,7 +1593,7 @@ static void s_s3_auto_ranged_put_request_finished( aws_replace_quote_entities(meta_request->allocator, etag_header_value); aws_http_headers_set( - final_response_headers, + request->send_data.response_headers, g_etag_header_name, aws_byte_cursor_from_buf(&etag_header_value_byte_buf)); @@ -1609,7 +1605,7 @@ static void s_s3_auto_ranged_put_request_finished( /* Notify the user of the headers. */ if (meta_request->headers_callback( meta_request, - final_response_headers, + request->send_data.response_headers, request->send_data.response_status, meta_request->user_data)) { @@ -1618,8 +1614,6 @@ static void s_s3_auto_ranged_put_request_finished( meta_request->headers_callback = NULL; /* Grab the lock again after the callback */ aws_s3_meta_request_lock_synced_data(meta_request); - - aws_http_headers_release(final_response_headers); } auto_ranged_put->synced_data.complete_multipart_upload_completed = true; diff --git a/source/s3_copy_object.c b/source/s3_copy_object.c index 15ed2ec0..86dec71a 100644 --- a/source/s3_copy_object.c +++ b/source/s3_copy_object.c @@ -603,17 +603,13 @@ static void s_s3_copy_object_request_finished( /* Invoke headers callback if it was requested for this meta request */ if (meta_request->headers_callback != NULL) { - struct aws_http_headers *final_response_headers = aws_http_headers_new(meta_request->allocator); - - /* Copy all the response headers from this request. */ - copy_http_headers(request->send_data.response_headers, final_response_headers); /* Invoke the callback without lock */ aws_s3_meta_request_unlock_synced_data(meta_request); /* Notify the user of the headers. */ if (meta_request->headers_callback( meta_request, - final_response_headers, + request->send_data.response_headers, request->send_data.response_status, meta_request->user_data)) { @@ -622,8 +618,6 @@ static void s_s3_copy_object_request_finished( meta_request->headers_callback = NULL; /* Grab the lock again after the callback */ aws_s3_meta_request_lock_synced_data(meta_request); - - aws_http_headers_release(final_response_headers); } /* Signals completion of the meta request */ @@ -741,15 +735,12 @@ static void s_s3_copy_object_request_finished( case AWS_S3_COPY_OBJECT_REQUEST_TAG_COMPLETE_MULTIPART_UPLOAD: { if (error_code == AWS_ERROR_SUCCESS && meta_request->headers_callback != NULL) { - struct aws_http_headers *final_response_headers = aws_http_headers_new(meta_request->allocator); - - /* Copy all the response headers from this request. */ - copy_http_headers(request->send_data.response_headers, final_response_headers); /* Copy over any response headers that we've previously determined are needed for this final * response. */ - copy_http_headers(copy_object->synced_data.needed_response_headers, final_response_headers); + copy_http_headers( + copy_object->synced_data.needed_response_headers, request->send_data.response_headers); struct aws_byte_cursor xml_doc = aws_byte_cursor_from_buf(&request->send_data.response_body); @@ -762,7 +753,7 @@ static void s_s3_copy_object_request_finished( aws_replace_quote_entities(meta_request->allocator, etag_header_value); aws_http_headers_set( - final_response_headers, + request->send_data.response_headers, g_etag_header_name, aws_byte_cursor_from_buf(&etag_header_value_byte_buf)); @@ -774,7 +765,7 @@ static void s_s3_copy_object_request_finished( aws_s3_meta_request_unlock_synced_data(meta_request); if (meta_request->headers_callback( meta_request, - final_response_headers, + request->send_data.response_headers, request->send_data.response_status, meta_request->user_data)) { @@ -783,8 +774,6 @@ static void s_s3_copy_object_request_finished( meta_request->headers_callback = NULL; /* Grab the lock again after the callback */ aws_s3_meta_request_lock_synced_data(meta_request); - - aws_http_headers_release(final_response_headers); } copy_object->synced_data.complete_multipart_upload_completed = true; diff --git a/source/s3_default_meta_request.c b/source/s3_default_meta_request.c index f1b43b79..4e576d43 100644 --- a/source/s3_default_meta_request.c +++ b/source/s3_default_meta_request.c @@ -384,18 +384,29 @@ static void s_s3_meta_request_default_request_finished( struct aws_s3_meta_request_default *meta_request_default = meta_request->impl; AWS_PRECONDITION(meta_request_default); - if (error_code == AWS_ERROR_SUCCESS && meta_request->headers_callback != NULL && - request->send_data.response_headers != NULL) { - - if (meta_request->headers_callback( - meta_request, - request->send_data.response_headers, - request->send_data.response_status, - meta_request->user_data)) { - error_code = aws_last_error_or_unknown(); + if (error_code == AWS_ERROR_SUCCESS && request->send_data.response_headers != NULL) { + if (meta_request->checksum_config.validate_response_checksum) { + if (aws_s3_check_headers_for_checksum( + meta_request, + request->send_data.response_headers, + &meta_request->meta_request_level_running_response_sum, + &meta_request->meta_request_level_response_header_checksum, + true) != AWS_OP_SUCCESS) { + error_code = aws_last_error_or_unknown(); + } } - meta_request->headers_callback = NULL; + if (error_code == AWS_ERROR_SUCCESS && meta_request->headers_callback != NULL) { + if (meta_request->headers_callback( + meta_request, + request->send_data.response_headers, + request->send_data.response_status, + meta_request->user_data)) { + error_code = aws_last_error_or_unknown(); + } + + meta_request->headers_callback = NULL; + } } /* BEGIN CRITICAL SECTION */ diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c index 06c99f8e..d3be24a1 100644 --- a/source/s3_meta_request.c +++ b/source/s3_meta_request.c @@ -94,92 +94,55 @@ void aws_s3_meta_request_unlock_synced_data(struct aws_s3_meta_request *meta_req aws_mutex_unlock(&meta_request->synced_data.lock); } -static int s_meta_request_get_response_headers_checksum_callback( - struct aws_s3_meta_request *meta_request, - const struct aws_http_headers *headers, - int response_status, - void *user_data) { - for (int i = AWS_SCA_INIT; i <= AWS_SCA_END; i++) { - if (!aws_s3_meta_request_checksum_config_has_algorithm(meta_request, i)) { - /* If user doesn't select this algorithm, skip */ - continue; - } - const struct aws_byte_cursor *algorithm_header_name = aws_get_http_header_name_from_algorithm(i); - if (aws_http_headers_has(headers, *algorithm_header_name) && - !aws_http_headers_has(headers, g_mp_parts_count_header_name)) { - struct aws_byte_cursor header_sum; - aws_http_headers_get(headers, *algorithm_header_name, &header_sum); - size_t encoded_len = 0; - aws_base64_compute_encoded_len(aws_get_digest_size_from_algorithm(i), &encoded_len); - if (header_sum.len == encoded_len - 1) { - /* encoded_len includes the nullptr length. -1 is the expected length. */ - aws_byte_buf_init_copy_from_cursor( - &meta_request->meta_request_level_response_header_checksum, meta_request->allocator, header_sum); - meta_request->meta_request_level_running_response_sum = aws_checksum_new(meta_request->allocator, i); - } - break; - } - } - if (meta_request->headers_user_callback_after_checksum) { - return meta_request->headers_user_callback_after_checksum(meta_request, headers, response_status, user_data); - } else { - return AWS_OP_SUCCESS; - } -} +/* True if the checksum validated and matched, false otherwise. */ +static bool s_validate_checksum( + struct aws_s3_checksum *checksum_to_validate, + struct aws_byte_buf *expected_encoded_checksum) { -/* warning this might get screwed up with retries/restarts */ -static int s_meta_request_get_response_body_checksum_callback( - struct aws_s3_meta_request *meta_request, - const struct aws_byte_cursor *body, - uint64_t range_start, - void *user_data) { - if (meta_request->meta_request_level_running_response_sum) { - aws_checksum_update(meta_request->meta_request_level_running_response_sum, body); + struct aws_byte_buf response_body_sum; + struct aws_byte_buf encoded_response_body_sum; + AWS_ZERO_STRUCT(response_body_sum); + AWS_ZERO_STRUCT(encoded_response_body_sum); + bool validated = false; + + size_t encoded_checksum_len = 0; + if (aws_base64_compute_encoded_len(checksum_to_validate->digest_size, &encoded_checksum_len)) { + goto done; } + aws_byte_buf_init(&encoded_response_body_sum, checksum_to_validate->allocator, encoded_checksum_len); + aws_byte_buf_init(&response_body_sum, checksum_to_validate->allocator, checksum_to_validate->digest_size); - if (meta_request->body_user_callback_after_checksum) { - return meta_request->body_user_callback_after_checksum(meta_request, body, range_start, user_data); - } else { - return AWS_OP_SUCCESS; + if (aws_checksum_finalize(checksum_to_validate, &response_body_sum, 0)) { + goto done; + } + struct aws_byte_cursor response_body_sum_cursor = aws_byte_cursor_from_buf(&response_body_sum); + if (aws_base64_encode(&response_body_sum_cursor, &encoded_response_body_sum)) { + goto done; } + if (aws_byte_buf_eq(&encoded_response_body_sum, expected_encoded_checksum)) { + validated = true; + } +done: + aws_byte_buf_clean_up(&response_body_sum); + aws_byte_buf_clean_up(&encoded_response_body_sum); + return validated; } -static void s_meta_request_get_response_finish_checksum_callback( +/* Prepare the finish request when we validate the checksum */ +static void s_validate_meta_request_checksum_on_finish( struct aws_s3_meta_request *meta_request, - const struct aws_s3_meta_request_result *meta_request_result, - void *user_data) { - struct aws_byte_buf response_body_sum; - struct aws_byte_buf encoded_response_body_sum; - AWS_ZERO_STRUCT(response_body_sum); - AWS_ZERO_STRUCT(encoded_response_body_sum); + struct aws_s3_meta_request_result *meta_request_result) { - struct aws_s3_meta_request_result *mut_meta_request_result = - (struct aws_s3_meta_request_result *)meta_request_result; if (meta_request_result->error_code == AWS_OP_SUCCESS && meta_request->meta_request_level_running_response_sum) { - mut_meta_request_result->did_validate = true; - mut_meta_request_result->validation_algorithm = - meta_request->meta_request_level_running_response_sum->algorithm; - size_t encoded_checksum_len = 0; - /* what error should I raise for these? */ - aws_base64_compute_encoded_len( - meta_request->meta_request_level_running_response_sum->digest_size, &encoded_checksum_len); - aws_byte_buf_init(&encoded_response_body_sum, meta_request->allocator, encoded_checksum_len); - aws_byte_buf_init( - &response_body_sum, - meta_request->allocator, - meta_request->meta_request_level_running_response_sum->digest_size); - aws_checksum_finalize(meta_request->meta_request_level_running_response_sum, &response_body_sum, 0); - struct aws_byte_cursor response_body_sum_cursor = aws_byte_cursor_from_buf(&response_body_sum); - aws_base64_encode(&response_body_sum_cursor, &encoded_response_body_sum); - if (!aws_byte_buf_eq(&encoded_response_body_sum, &meta_request->meta_request_level_response_header_checksum)) { - mut_meta_request_result->error_code = AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH; + meta_request_result->did_validate = true; + meta_request_result->validation_algorithm = meta_request->meta_request_level_running_response_sum->algorithm; + if (!s_validate_checksum( + meta_request->meta_request_level_running_response_sum, + &meta_request->meta_request_level_response_header_checksum)) { + meta_request_result->error_code = AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH; + AWS_LOGF_ERROR(AWS_LS_S3_META_REQUEST, "id=%p Checksum mismatch!", (void *)meta_request); } } - if (meta_request->finish_user_callback_after_checksum) { - meta_request->finish_user_callback_after_checksum(meta_request, meta_request_result, user_data); - } - aws_byte_buf_clean_up(&response_body_sum); - aws_byte_buf_clean_up(&encoded_response_body_sum); aws_checksum_destroy(meta_request->meta_request_level_running_response_sum); aws_byte_buf_clean_up(&meta_request->meta_request_level_response_header_checksum); } @@ -294,20 +257,9 @@ int aws_s3_meta_request_init_base( meta_request->telemetry_callback = options->telemetry_callback; meta_request->upload_review_callback = options->upload_review_callback; - if (meta_request->checksum_config.validate_response_checksum) { - /* TODO: the validate for auto range get should happen for each response received. */ - meta_request->headers_user_callback_after_checksum = options->headers_callback; - meta_request->body_user_callback_after_checksum = options->body_callback; - meta_request->finish_user_callback_after_checksum = options->finish_callback; - - meta_request->headers_callback = s_meta_request_get_response_headers_checksum_callback; - meta_request->body_callback = s_meta_request_get_response_body_checksum_callback; - meta_request->finish_callback = s_meta_request_get_response_finish_checksum_callback; - } else { - meta_request->headers_callback = options->headers_callback; - meta_request->body_callback = options->body_callback; - meta_request->finish_callback = options->finish_callback; - } + meta_request->headers_callback = options->headers_callback; + meta_request->body_callback = options->body_callback; + meta_request->finish_callback = options->finish_callback; /* Nothing can fail after here. Leave the impl not affected by failure of initializing base. */ meta_request->impl = impl; @@ -1201,46 +1153,6 @@ static void s_get_part_response_headers_checksum_helper( } } -/* warning this might get screwed up with retries/restarts */ -static void s_get_part_response_body_checksum_helper( - struct aws_s3_checksum *running_response_sum, - const struct aws_byte_cursor *body) { - if (running_response_sum) { - aws_checksum_update(running_response_sum, body); - } -} - -static void s_get_response_part_finish_checksum_helper(struct aws_s3_connection *connection, int error_code) { - struct aws_byte_buf response_body_sum; - struct aws_byte_buf encoded_response_body_sum; - AWS_ZERO_STRUCT(response_body_sum); - AWS_ZERO_STRUCT(encoded_response_body_sum); - - struct aws_s3_request *request = connection->request; - if (error_code == AWS_OP_SUCCESS && request->request_level_running_response_sum) { - size_t encoded_checksum_len = 0; - request->did_validate = true; - aws_base64_compute_encoded_len(request->request_level_running_response_sum->digest_size, &encoded_checksum_len); - aws_byte_buf_init(&encoded_response_body_sum, request->allocator, encoded_checksum_len); - aws_byte_buf_init( - &response_body_sum, request->allocator, request->request_level_running_response_sum->digest_size); - aws_checksum_finalize(request->request_level_running_response_sum, &response_body_sum, 0); - struct aws_byte_cursor response_body_sum_cursor = aws_byte_cursor_from_buf(&response_body_sum); - aws_base64_encode(&response_body_sum_cursor, &encoded_response_body_sum); - request->checksum_match = - aws_byte_buf_eq(&encoded_response_body_sum, &request->request_level_response_header_checksum); - - request->validation_algorithm = request->request_level_running_response_sum->algorithm; - aws_byte_buf_clean_up(&response_body_sum); - aws_byte_buf_clean_up(&encoded_response_body_sum); - } else { - request->did_validate = false; - } - aws_checksum_destroy(request->request_level_running_response_sum); - aws_byte_buf_clean_up(&request->request_level_response_header_checksum); - request->request_level_running_response_sum = NULL; -} - static int s_s3_meta_request_incoming_headers( struct aws_http_stream *stream, enum aws_http_header_block header_block, @@ -1292,6 +1204,7 @@ static int s_s3_meta_request_incoming_headers( if (successful_response && meta_request->checksum_config.validate_response_checksum && request->request_type == AWS_S3_REQUEST_TYPE_GET_OBJECT) { + /* We have `struct aws_http_header *` array instead of `struct aws_http_headers *` :) */ s_get_part_response_headers_checksum_helper(connection, meta_request, headers, headers_count); } @@ -1387,8 +1300,9 @@ static int s_s3_meta_request_incoming_body( AWS_LOGF_TRACE(AWS_LS_S3_META_REQUEST, "response body: \n" PRInSTR "\n", AWS_BYTE_CURSOR_PRI(*data)); } - if (meta_request->checksum_config.validate_response_checksum) { - s_get_part_response_body_checksum_helper(request->request_level_running_response_sum, data); + if (meta_request->checksum_config.validate_response_checksum && request->request_level_running_response_sum) { + /* Update the request level checksum. */ + aws_checksum_update(request->request_level_running_response_sum, data); } if (request->send_data.response_body.capacity == 0) { @@ -1459,7 +1373,27 @@ static void s_s3_meta_request_stream_complete(struct aws_http_stream *stream, in struct aws_s3_meta_request *meta_request = request->meta_request; if (meta_request->checksum_config.validate_response_checksum) { - s_get_response_part_finish_checksum_helper(connection, error_code); + /* finish the request level checksum validation. */ + if (error_code == AWS_OP_SUCCESS && request->request_level_running_response_sum) { + request->did_validate = true; + request->validation_algorithm = request->request_level_running_response_sum->algorithm; + request->checksum_match = s_validate_checksum( + request->request_level_running_response_sum, &request->request_level_response_header_checksum); + if (!request->checksum_match) { + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, + "id=%p Checksum mismatch! (request=%p, response status=%d)", + (void *)meta_request, + (void *)request, + request->send_data.response_status); + error_code = AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH; + } + } else { + request->did_validate = false; + } + aws_checksum_destroy(request->request_level_running_response_sum); + aws_byte_buf_clean_up(&request->request_level_response_header_checksum); + request->request_level_running_response_sum = NULL; } /* BEGIN CRITICAL SECTION */ { @@ -1594,20 +1528,7 @@ void aws_s3_meta_request_send_request_finish_default( enum aws_s3_connection_finish_code finish_code = AWS_S3_CONNECTION_FINISH_CODE_FAILED; if (error_code == AWS_ERROR_SUCCESS) { - if (connection->request->meta_request->type == AWS_S3_META_REQUEST_TYPE_GET_OBJECT && request->did_validate && - !request->checksum_match) { - finish_code = AWS_S3_CONNECTION_FINISH_CODE_FAILED; - error_code = AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH; - AWS_LOGF_ERROR( - AWS_LS_S3_META_REQUEST, - "id=%p Meta request cannot recover from checksum mismatch. (request=%p, response status=%d)", - (void *)meta_request, - (void *)request, - response_status); - } else { - finish_code = AWS_S3_CONNECTION_FINISH_CODE_SUCCESS; - } - + finish_code = AWS_S3_CONNECTION_FINISH_CODE_SUCCESS; } else { /* BEGIN CRITICAL SECTION */ aws_s3_meta_request_lock_synced_data(meta_request); @@ -1619,7 +1540,8 @@ void aws_s3_meta_request_send_request_finish_default( * has a result, then make sure that this request isn't retried. */ if (error_code == AWS_ERROR_S3_INVALID_RESPONSE_STATUS || error_code == AWS_ERROR_S3_INTERNAL_PART_SIZE_MISMATCH_RETRYING_WITH_RANGE || - error_code == AWS_ERROR_S3_NON_RECOVERABLE_ASYNC_ERROR || meta_request_finishing) { + error_code == AWS_ERROR_S3_NON_RECOVERABLE_ASYNC_ERROR || + error_code == AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH || meta_request_finishing) { finish_code = AWS_S3_CONNECTION_FINISH_CODE_FAILED; if (error_code == AWS_ERROR_S3_INTERNAL_PART_SIZE_MISMATCH_RETRYING_WITH_RANGE) { /* Log at info level instead of error as it's expected and not a fatal error */ @@ -1855,8 +1777,12 @@ static void s_s3_meta_request_event_delivery_task(struct aws_task *task, void *a AWS_ASSERT(request->part_number >= 1); - if (error_code == AWS_ERROR_SUCCESS && response_body.len > 0 && meta_request->body_callback != NULL) { - if (meta_request->body_callback( + if (error_code == AWS_ERROR_SUCCESS && response_body.len > 0) { + if (meta_request->meta_request_level_running_response_sum) { + aws_checksum_update(meta_request->meta_request_level_running_response_sum, &response_body); + } + if (meta_request->body_callback != NULL && + meta_request->body_callback( meta_request, &response_body, request->part_range_start, meta_request->user_data)) { error_code = aws_last_error_or_unknown(); @@ -2089,6 +2015,10 @@ void aws_s3_meta_request_finish_default(struct aws_s3_meta_request *meta_request meta_request->request_body_parallel_stream = aws_parallel_input_stream_release(meta_request->request_body_parallel_stream); meta_request->initial_request_message = aws_http_message_release(meta_request->initial_request_message); + if (meta_request->checksum_config.validate_response_checksum) { + /* validate checksum finish */ + s_validate_meta_request_checksum_on_finish(meta_request, &finish_result); + } if (meta_request->finish_callback != NULL) { meta_request->finish_callback(meta_request, &finish_result, meta_request->user_data); diff --git a/source/s3_util.c b/source/s3_util.c index 5eaa9118..106f29be 100644 --- a/source/s3_util.c +++ b/source/s3_util.c @@ -10,6 +10,7 @@ #include "aws/s3/private/s3_request.h" #include #include +#include #include #include #include @@ -709,3 +710,58 @@ void aws_s3_request_finish_up_metrics_synced(struct aws_s3_request *request, str request->send_data.metrics = aws_s3_request_metrics_release(metrics); } } + +int aws_s3_check_headers_for_checksum( + struct aws_s3_meta_request *meta_request, + const struct aws_http_headers *headers, + struct aws_s3_checksum **out_checksum, + struct aws_byte_buf *out_checksum_buffer, + bool meta_request_level) { + AWS_PRECONDITION(meta_request); + AWS_PRECONDITION(out_checksum); + AWS_PRECONDITION(out_checksum_buffer); + + if (!headers || aws_http_headers_count(headers) == 0) { + *out_checksum = NULL; + return AWS_OP_SUCCESS; + } + if (meta_request_level && aws_http_headers_has(headers, g_mp_parts_count_header_name)) { + /* g_mp_parts_count_header_name indicates it's a object was uploaded as a + * multipart upload. So, the checksum should not be applied to the meta request level. + * But we we want to check it for the request level. */ + *out_checksum = NULL; + return AWS_OP_SUCCESS; + } + + for (int i = AWS_SCA_INIT; i <= AWS_SCA_END; i++) { + if (!aws_s3_meta_request_checksum_config_has_algorithm(meta_request, i)) { + /* If user doesn't select this algorithm, skip */ + continue; + } + const struct aws_byte_cursor *algorithm_header_name = aws_get_http_header_name_from_algorithm(i); + struct aws_byte_cursor checksum_value; + if (aws_http_headers_get(headers, *algorithm_header_name, &checksum_value) == AWS_OP_SUCCESS) { + /* Found the checksum header, keep the header value and initialize the running checksum */ + size_t encoded_len = 0; + aws_base64_compute_encoded_len(aws_get_digest_size_from_algorithm(i), &encoded_len); + if (checksum_value.len == encoded_len - 1) { + /* encoded_len includes the nullptr length. -1 is the expected length. */ + aws_byte_buf_init_copy_from_cursor(out_checksum_buffer, meta_request->allocator, checksum_value); + *out_checksum = aws_checksum_new(meta_request->allocator, i); + if (!*out_checksum) { + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, + "Could not create checksum for algorithm: %d, due to error code %d (%s)", + i, + aws_last_error_or_unknown(), + aws_error_str(aws_last_error_or_unknown())); + return AWS_OP_ERR; + } + return AWS_OP_SUCCESS; + } + break; + } + } + *out_checksum = NULL; + return AWS_OP_SUCCESS; +}