Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rewrite some checksum related code #443

Merged
merged 9 commits into from
Aug 15, 2024
Merged

rewrite some checksum related code #443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8045073
rewrite some checksum related code
TingDaoK Jun 21, 2024
51185c6
it's just getting worse
TingDaoK Jun 21, 2024
6231c60
format
TingDaoK Jun 21, 2024
aca9b76
check for error
TingDaoK Jun 24, 2024
ce75d8c
add a log
TingDaoK Jun 24, 2024
7311370
Merge branch 'checksum-callback' of github.com:awslabs/aws-c-s3 into …
TingDaoK Jun 24, 2024
b18dc5e
pointer to pointer..
TingDaoK Jun 24, 2024
ccb46ee
use one function instead of two
TingDaoK Jun 24, 2024
6698fc9
Merge branch 'main' into checksum-callback
TingDaoK Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions include/aws/s3/private/s3_meta_request_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,11 +176,6 @@ struct aws_s3_meta_request {
aws_s3_meta_request_telemetry_fn *telemetry_callback;
aws_s3_meta_request_upload_review_fn *upload_review_callback;

/* Customer specified callbacks to be called by our specialized callback to calculate the response checksum. */
aws_s3_meta_request_headers_callback_fn *headers_user_callback_after_checksum;
aws_s3_meta_request_receive_body_callback_fn *body_user_callback_after_checksum;
aws_s3_meta_request_finish_fn *finish_user_callback_after_checksum;

enum aws_s3_meta_request_type type;
struct aws_string *s3express_session_host;

Expand Down
11 changes: 11 additions & 0 deletions include/aws/s3/private/s3_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ struct aws_http_message;
struct aws_s3_client;
struct aws_s3_request;
struct aws_s3_meta_request;
struct aws_s3_checksum;

struct aws_cached_signing_config_aws {
struct aws_allocator *allocator;
Expand Down Expand Up @@ -323,6 +324,16 @@ int aws_s3_crt_error_code_from_recoverable_server_error_code_string(struct aws_b
AWS_S3_API
void aws_s3_request_finish_up_metrics_synced(struct aws_s3_request *request, struct aws_s3_meta_request *meta_request);

/* Check the response headers for checksum to verify, return a running checksum based on the algorithm found. If no
* checksum found from header, return null. */
AWS_S3_API
int aws_s3_check_headers_for_checksum(
struct aws_s3_meta_request *meta_request,
const struct aws_http_headers *headers,
struct aws_s3_checksum **out_checksum,
struct aws_byte_buf *out_checksum_buffer,
bool meta_request_level);

AWS_EXTERN_C_END

#endif /* AWS_S3_UTIL_H */
30 changes: 20 additions & 10 deletions source/s3_auto_ranged_get.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -809,11 +809,21 @@ static void s_s3_auto_ranged_get_request_finished(
error_code = AWS_ERROR_SUCCESS;
found_object_size = true;

if (!empty_file_error && meta_request->headers_callback != NULL) {
struct aws_http_headers *response_headers = aws_http_headers_new(meta_request->allocator);

copy_http_headers(request->send_data.response_headers, response_headers);
/* Check for checksums if requested to */
if (meta_request->checksum_config.validate_response_checksum) {
if (aws_s3_check_headers_for_checksum(
meta_request,
request->send_data.response_headers,
&meta_request->meta_request_level_running_response_sum,
&meta_request->meta_request_level_response_header_checksum,
true) != AWS_OP_SUCCESS) {
error_code = aws_last_error_or_unknown();
goto update_synced_data;
}
}

if (!empty_file_error && meta_request->headers_callback != NULL) {
/* Modify the header received to fake the header for the whole meta request. */
if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_GET_OBJECT_WITH_RANGE ||
request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_GET_OBJECT_WITH_PART_NUMBER_1) {

Expand All @@ -828,32 +838,32 @@ static void s_s3_auto_ranged_get_request_finished(
object_range_end,
object_size);
aws_http_headers_set(
response_headers,
request->send_data.response_headers,
g_content_range_header_name,
aws_byte_cursor_from_c_str(content_range_buffer));
} else {
/* content range isn't applicable. */
aws_http_headers_erase(response_headers, g_content_range_header_name);
aws_http_headers_erase(request->send_data.response_headers, g_content_range_header_name);
}
}

uint64_t content_length = object_size ? object_range_end - object_range_start + 1 : 0;
char content_length_buffer[64] = "";
snprintf(content_length_buffer, sizeof(content_length_buffer), "%" PRIu64, content_length);
aws_http_headers_set(
response_headers, g_content_length_header_name, aws_byte_cursor_from_c_str(content_length_buffer));
request->send_data.response_headers,
g_content_length_header_name,
aws_byte_cursor_from_c_str(content_length_buffer));

if (meta_request->headers_callback(
meta_request,
response_headers,
request->send_data.response_headers,
s_s3_auto_ranged_get_success_status(meta_request),
meta_request->user_data)) {

error_code = aws_last_error_or_unknown();
}
meta_request->headers_callback = NULL;

aws_http_headers_release(response_headers);
}
}

Expand Down
14 changes: 4 additions & 10 deletions source/s3_auto_ranged_put.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1568,15 +1568,11 @@ static void s_s3_auto_ranged_put_request_finished(

case AWS_S3_AUTO_RANGED_PUT_REQUEST_TAG_COMPLETE_MULTIPART_UPLOAD: {
if (error_code == AWS_ERROR_SUCCESS && meta_request->headers_callback != NULL) {
struct aws_http_headers *final_response_headers = aws_http_headers_new(meta_request->allocator);

/* Copy all the response headers from this request. */
copy_http_headers(request->send_data.response_headers, final_response_headers);

/* Copy over any response headers that we've previously determined are needed for this final
* response.
*/
copy_http_headers(auto_ranged_put->synced_data.needed_response_headers, final_response_headers);
copy_http_headers(
auto_ranged_put->synced_data.needed_response_headers, request->send_data.response_headers);

struct aws_byte_cursor xml_doc = aws_byte_cursor_from_buf(&request->send_data.response_body);

Expand All @@ -1597,7 +1593,7 @@ static void s_s3_auto_ranged_put_request_finished(
aws_replace_quote_entities(meta_request->allocator, etag_header_value);

aws_http_headers_set(
final_response_headers,
request->send_data.response_headers,
g_etag_header_name,
aws_byte_cursor_from_buf(&etag_header_value_byte_buf));

Expand All @@ -1609,7 +1605,7 @@ static void s_s3_auto_ranged_put_request_finished(
/* Notify the user of the headers. */
if (meta_request->headers_callback(
meta_request,
final_response_headers,
request->send_data.response_headers,
request->send_data.response_status,
meta_request->user_data)) {

Expand All @@ -1618,8 +1614,6 @@ static void s_s3_auto_ranged_put_request_finished(
meta_request->headers_callback = NULL;
/* Grab the lock again after the callback */
aws_s3_meta_request_lock_synced_data(meta_request);

aws_http_headers_release(final_response_headers);
}

auto_ranged_put->synced_data.complete_multipart_upload_completed = true;
Expand Down
21 changes: 5 additions & 16 deletions source/s3_copy_object.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -603,17 +603,13 @@ static void s_s3_copy_object_request_finished(

/* Invoke headers callback if it was requested for this meta request */
if (meta_request->headers_callback != NULL) {
struct aws_http_headers *final_response_headers = aws_http_headers_new(meta_request->allocator);

/* Copy all the response headers from this request. */
copy_http_headers(request->send_data.response_headers, final_response_headers);

/* Invoke the callback without lock */
aws_s3_meta_request_unlock_synced_data(meta_request);
/* Notify the user of the headers. */
if (meta_request->headers_callback(
meta_request,
final_response_headers,
request->send_data.response_headers,
request->send_data.response_status,
meta_request->user_data)) {

Expand All @@ -622,8 +618,6 @@ static void s_s3_copy_object_request_finished(
meta_request->headers_callback = NULL;
/* Grab the lock again after the callback */
aws_s3_meta_request_lock_synced_data(meta_request);

aws_http_headers_release(final_response_headers);
}

/* Signals completion of the meta request */
Expand Down Expand Up @@ -741,15 +735,12 @@ static void s_s3_copy_object_request_finished(

case AWS_S3_COPY_OBJECT_REQUEST_TAG_COMPLETE_MULTIPART_UPLOAD: {
if (error_code == AWS_ERROR_SUCCESS && meta_request->headers_callback != NULL) {
struct aws_http_headers *final_response_headers = aws_http_headers_new(meta_request->allocator);

/* Copy all the response headers from this request. */
copy_http_headers(request->send_data.response_headers, final_response_headers);

/* Copy over any response headers that we've previously determined are needed for this final
* response.
*/
copy_http_headers(copy_object->synced_data.needed_response_headers, final_response_headers);
copy_http_headers(
copy_object->synced_data.needed_response_headers, request->send_data.response_headers);

struct aws_byte_cursor xml_doc = aws_byte_cursor_from_buf(&request->send_data.response_body);

Expand All @@ -762,7 +753,7 @@ static void s_s3_copy_object_request_finished(
aws_replace_quote_entities(meta_request->allocator, etag_header_value);

aws_http_headers_set(
final_response_headers,
request->send_data.response_headers,
g_etag_header_name,
aws_byte_cursor_from_buf(&etag_header_value_byte_buf));

Expand All @@ -774,7 +765,7 @@ static void s_s3_copy_object_request_finished(
aws_s3_meta_request_unlock_synced_data(meta_request);
if (meta_request->headers_callback(
meta_request,
final_response_headers,
request->send_data.response_headers,
request->send_data.response_status,
meta_request->user_data)) {

Expand All @@ -783,8 +774,6 @@ static void s_s3_copy_object_request_finished(
meta_request->headers_callback = NULL;
/* Grab the lock again after the callback */
aws_s3_meta_request_lock_synced_data(meta_request);

aws_http_headers_release(final_response_headers);
}

copy_object->synced_data.complete_multipart_upload_completed = true;
Expand Down
31 changes: 21 additions & 10 deletions source/s3_default_meta_request.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,18 +384,29 @@ static void s_s3_meta_request_default_request_finished(
struct aws_s3_meta_request_default *meta_request_default = meta_request->impl;
AWS_PRECONDITION(meta_request_default);

if (error_code == AWS_ERROR_SUCCESS && meta_request->headers_callback != NULL &&
request->send_data.response_headers != NULL) {

if (meta_request->headers_callback(
meta_request,
request->send_data.response_headers,
request->send_data.response_status,
meta_request->user_data)) {
error_code = aws_last_error_or_unknown();
if (error_code == AWS_ERROR_SUCCESS && request->send_data.response_headers != NULL) {
if (meta_request->checksum_config.validate_response_checksum) {
if (aws_s3_check_headers_for_checksum(
meta_request,
request->send_data.response_headers,
&meta_request->meta_request_level_running_response_sum,
&meta_request->meta_request_level_response_header_checksum,
true) != AWS_OP_SUCCESS) {
error_code = aws_last_error_or_unknown();
}
}

meta_request->headers_callback = NULL;
if (error_code == AWS_ERROR_SUCCESS && meta_request->headers_callback != NULL) {
if (meta_request->headers_callback(
meta_request,
request->send_data.response_headers,
request->send_data.response_status,
meta_request->user_data)) {
error_code = aws_last_error_or_unknown();
}

meta_request->headers_callback = NULL;
}
}

/* BEGIN CRITICAL SECTION */
Expand Down
Loading