Skip to content

Commit

Permalink
ignore the checksum and skip body callback
Browse files Browse the repository at this point in the history
  • Loading branch information
@TingDaoK
TingDaoK committed Aug 19, 2024
1 parent f924302 commit e058c2b
Show file tree
Hide file tree
Showing 5 changed files with 11 additions and 36 deletions.
2 changes: 1 addition & 1 deletion include/aws/s3/s3_client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -657,7 +657,7 @@ struct aws_s3_meta_request_options {
/**
* Optional.
* If set, the received data will be written into this file.
* the `body_callback` will still be invoked if set.
* the `body_callback` will NOT be invoked.
* This gives a better performance when receiving data to write to a file.
*/
struct aws_byte_cursor receive_filepath;
Expand Down
4 changes: 2 additions & 2 deletions source/s3_meta_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1850,8 +1850,8 @@ static void s_s3_meta_request_event_delivery_task(struct aws_task *task, void *a
errno_value,
aws_error_name(aws_last_error()));
}
}
if (meta_request->body_callback != NULL &&
} else if (
meta_request->body_callback != NULL &&
meta_request->body_callback(
meta_request, &response_body, request->part_range_start, meta_request->user_data)) {

Expand Down
7 changes: 4 additions & 3 deletions tests/s3_data_plane_tests.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1525,7 +1525,7 @@ static int s_test_s3_get_object_file_path_append(struct aws_allocator *allocator

ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, &meta_request_test_results));
ASSERT_UINT_EQUALS(pre_exist_file_length + MB_TO_BYTES(1), meta_request_test_results.received_file_size);
ASSERT_UINT_EQUALS(MB_TO_BYTES(1), meta_request_test_results.received_body_size);
ASSERT_UINT_EQUALS(MB_TO_BYTES(1), meta_request_test_results.progress.total_bytes_transferred);
aws_s3_meta_request_test_results_clean_up(&meta_request_test_results);
client = aws_s3_client_release(client);
aws_s3_tester_clean_up(&tester);
Expand Down Expand Up @@ -1582,7 +1582,7 @@ static int s_test_s3_get_object_file_path_to_position(struct aws_allocator *allo
ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, &meta_request_test_results));
ASSERT_UINT_EQUALS(
get_options.get_options.recv_file_position + MB_TO_BYTES(1), meta_request_test_results.received_file_size);
ASSERT_UINT_EQUALS(MB_TO_BYTES(1), meta_request_test_results.received_body_size);
ASSERT_UINT_EQUALS(MB_TO_BYTES(1), meta_request_test_results.progress.total_bytes_transferred);

aws_s3_meta_request_test_results_clean_up(&meta_request_test_results);
client = aws_s3_client_release(client);
Expand Down Expand Up @@ -4432,7 +4432,8 @@ static int s_test_s3_round_trip_with_filepath_helper(
};
ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, &test_results));

ASSERT_UINT_EQUALS(MB_TO_BYTES(put_options.put_options.object_size_mb), test_results.received_body_size);
ASSERT_UINT_EQUALS(
MB_TO_BYTES(put_options.put_options.object_size_mb), test_results.progress.total_bytes_transferred);

aws_s3_meta_request_test_results_clean_up(&test_results);
aws_byte_buf_clean_up(&path_buf);
Expand Down
32 changes: 4 additions & 28 deletions tests/s3_tester.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,6 @@ static int s_s3_test_meta_request_body_callback(
struct aws_s3_meta_request_test_results *meta_request_test_results = user_data;
meta_request_test_results->received_body_size += body->len;
aws_atomic_fetch_add(&meta_request_test_results->received_body_size_delta, body->len);
aws_checksum_update(meta_request_test_results->get_object_checksum_crc32c, body);

AWS_LOGF_DEBUG(
AWS_LS_S3_GENERAL,
"Received range %" PRIu64 "-%" PRIu64 ". Expected range start: %" PRIu64,
Expand Down Expand Up @@ -541,14 +539,12 @@ void aws_s3_meta_request_test_results_init(
aws_atomic_init_int(&test_meta_request->received_body_size_delta, 0);
aws_array_list_init_dynamic(
&test_meta_request->synced_data.metrics, allocator, 4, sizeof(struct aws_s3_request_metrics *));
test_meta_request->get_object_checksum_crc32c = aws_checksum_new(allocator, AWS_SCA_CRC32C);
}

void aws_s3_meta_request_test_results_clean_up(struct aws_s3_meta_request_test_results *test_meta_request) {
if (test_meta_request == NULL) {
return;
}
aws_checksum_destroy(test_meta_request->get_object_checksum_crc32c);
aws_http_headers_release(test_meta_request->error_response_headers);
aws_byte_buf_clean_up(&test_meta_request->error_response_body);
aws_string_destroy(test_meta_request->error_response_operation_name);
Expand Down Expand Up @@ -1781,27 +1777,7 @@ int aws_s3_tester_send_meta_request_with_options(
ASSERT_NOT_NULL(file);
ASSERT_SUCCESS(aws_file_get_length(file, &out_results->received_file_size));
if (options->get_options.recv_file_options == AWS_RECV_FILE_CREATE_OR_REPLACE) {
/* Only check the checksum when we create or replace the old file. */
uint8_t output_from_stream[4] = {0};
struct aws_byte_buf output_from_stream_buf =
aws_byte_buf_from_array(output_from_stream, sizeof(output_from_stream));
output_from_stream_buf.len = 0;
ASSERT_SUCCESS(
aws_checksum_finalize(out_results->get_object_checksum_crc32c, &output_from_stream_buf, 0));
struct aws_byte_buf buf;
aws_byte_buf_init(&buf, allocator, (size_t)out_results->received_file_size);
size_t read_length = fread(buf.buffer, 1, (size_t)out_results->received_file_size, file);
ASSERT_INT_EQUALS(out_results->received_file_size, (int64_t)read_length);
buf.len = read_length;
struct aws_byte_cursor file_cursor = aws_byte_cursor_from_buf(&buf);
uint8_t output_from_file[4] = {0};
struct aws_byte_buf output_from_file_buf =
aws_byte_buf_from_array(output_from_file, sizeof(output_from_file));
output_from_file_buf.len = 0;
ASSERT_SUCCESS(
aws_checksum_compute(allocator, AWS_SCA_CRC32C, &file_cursor, &output_from_file_buf, 0));
ASSERT_TRUE(aws_byte_buf_eq(&output_from_stream_buf, &output_from_file_buf));
aws_byte_buf_clean_up(&buf);
ASSERT_UINT_EQUALS(out_results->progress.total_bytes_transferred, out_results->received_file_size);
}
fclose(file);
}
Expand Down Expand Up @@ -1842,7 +1818,7 @@ int aws_s3_tester_send_meta_request_with_options(
aws_uri_clean_up(&mock_server);

if (filepath_str) {
// aws_file_delete(filepath_str);
aws_file_delete(filepath_str);
aws_string_destroy(filepath_str);
}

Expand Down Expand Up @@ -1988,9 +1964,9 @@ int aws_s3_tester_validate_get_object_results(
AWS_LS_S3_GENERAL,
"Content length in header is %" PRIu64 " and received body size is %" PRIu64,
content_length,
meta_request_test_results->received_body_size);
meta_request_test_results->progress.total_bytes_transferred);

ASSERT_TRUE(content_length == meta_request_test_results->received_body_size);
ASSERT_TRUE(content_length == meta_request_test_results->progress.total_bytes_transferred);
ASSERT_UINT_EQUALS(content_length, meta_request_test_results->progress.total_bytes_transferred);
ASSERT_UINT_EQUALS(content_length, meta_request_test_results->progress.content_length);

Expand Down
2 changes: 0 additions & 2 deletions tests/s3_tester.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,6 @@ struct aws_s3_meta_request_test_results {
int finished_error_code;
enum aws_s3_checksum_algorithm algorithm;

struct aws_s3_checksum *get_object_checksum_crc32c;

/* Record data from progress_callback() */
struct {
uint64_t content_length; /* Remember progress->content_length */
Expand Down

0 comments on commit e058c2b

Please sign in to comment.