Merge branch 'main' into python3.13

README.md

@@ -45,10 +45,10 @@ To simplify installation, aws-crt-python has its own copy of libcrypto.
 This lets you install a wheel from PyPI without having OpenSSL installed.
 Unix wheels on PyPI come with libcrypto statically compiled in.
 Code to build libcrypto comes from [AWS-LC](https://github.com/aws/aws-lc).
-AWS-LC's code is included in the PyPI source package, 
+AWS-LC's code is included in the PyPI source package,
 and the git repository includes it as a submodule.
 
-If you need aws-crt-python to use the libcrypto included on your system, 
+If you need aws-crt-python to use the libcrypto included on your system,
 set environment variable `AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=1` while building from source:
 
 ```sh
@@ -59,6 +59,19 @@ AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=1 python3 -m pip install --no-binary :all: --
 You can ignore all this on Windows and Apple platforms, where aws-crt-python
 uses the OS's default libraries for TLS and cryptography math.
 
+### AWS_CRT_BUILD_USE_SYSTEM_LIBS ###
+
+aws-crt-python depends on several C libraries that make up the AWS Common Runtime (libaws-c-common, libaws-c-s3, etc).
+By default, these libraries are built along with aws-crt-python and statically compiled in
+(their source code is under [crt/](crt/)).
+
+To skip building these dependencies, because they're already available on your system,
+set environment variable `AWS_CRT_BUILD_USE_SYSTEM_LIBS=1` while building from source:
+
+```sh
+AWS_CRT_BUILD_USE_SYSTEM_LIBS=1 python3 -m pip install .
+```
+
 ## Mac-Only TLS Behavior
 
 Please note that on Mac, once a private key is used with a certificate, that certificate-key pair is imported into the Mac Keychain. All subsequent uses of that certificate will use the stored private key and ignore anything passed in programmatically. Beginning in v0.6.2, when a stored private key from the Keychain is used, the following will be logged at the "info" log level:

awscrt/auth.py

@@ -507,7 +507,7 @@ class AwsSignedBodyHeaderType(IntEnum):
     """Do not add a header."""
 
     X_AMZ_CONTENT_SHA_256 = 1
-    """Add the "x-amz-content-sha-256" header with the canonical request's signed body value"""
+    """Add the "x-amz-content-sha256" header with the canonical request's signed body value"""
 
 
 class AwsSigningConfig(NativeResource):

awscrt/checksums.py

@@ -21,3 +21,12 @@ def crc32c(input: bytes, previous_crc32c: int = 0) -> int:
     Returns an unsigned 32-bit integer.
     """
     return _awscrt.checksums_crc32c(input, previous_crc32c)
+
+
+def crc64nvme(input: bytes, previous_crc64nvme: int = 0) -> int:
+    """
+    Perform a CRC64 NVME computation.
+    If continuing to update a running CRC, pass its value into `previous_crc64nvme`.
+    Returns an unsigned 64-bit integer.
+    """
+    return _awscrt.checksums_crc64nvme(input, previous_crc64nvme)

setup.py

@@ -134,8 +134,14 @@ def get_cmake_path():
     raise Exception("CMake must be installed to build from source.")
 
 
+def using_system_libs():
+    """If true, don't build any dependencies. Use the libs that are already on the system."""
+    return os.getenv('AWS_CRT_BUILD_USE_SYSTEM_LIBS') == '1'
+
+
 def using_system_libcrypto():
-    return os.getenv('AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO') == '1'
+    """If true, don't build AWS-LC. Use the libcrypto that's already on the system."""
+    return using_system_libs() or os.getenv('AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO') == '1'
 
 
 class AwsLib:
@@ -223,7 +229,10 @@ def _build_dependencies_impl(self, build_dir, install_path, osx_arch=None):
         ]
         run_cmd(build_cmd)
 
-    def _build_dependencies(self, build_dir, install_path):
+    def _build_dependencies(self):
+        build_dir = os.path.join(self.build_temp, 'deps')
+        install_path = os.path.join(self.build_temp, 'deps', 'install')
+
         if is_macos_universal2() and not is_development_mode():
             # create macOS universal binary by compiling for x86_64 and arm64,
             # each in its own subfolder, and then creating a universal binary
@@ -266,30 +275,28 @@ def _build_dependencies(self, build_dir, install_path):
             # normal build for a single architecture
             self._build_dependencies_impl(build_dir, install_path)
 
-    def run(self):
-        # build dependencies
-        dep_build_dir = os.path.join(self.build_temp, 'deps')
-        dep_install_path = os.path.join(self.build_temp, 'deps', 'install')
-
-        if os.path.exists(os.path.join(PROJECT_DIR, 'crt', 'aws-c-common', 'CMakeLists.txt')):
-            self._build_dependencies(dep_build_dir, dep_install_path)
-        else:
-            print("Skip building dependencies, source not found.")
-
         # update paths so awscrt_ext can access dependencies.
         # add to the front of any list so that our dependencies are preferred
         # over anything that might already be on the system (i.e. libcrypto.a)
 
-        self.include_dirs.insert(0, os.path.join(dep_install_path, 'include'))
+        self.include_dirs.insert(0, os.path.join(install_path, 'include'))
 
         # some platforms (ex: fedora) use /lib64 instead of just /lib
         lib_dir = 'lib'
-        if is_64bit() and os.path.exists(os.path.join(dep_install_path, 'lib64')):
+        if is_64bit() and os.path.exists(os.path.join(install_path, 'lib64')):
             lib_dir = 'lib64'
-        if is_32bit() and os.path.exists(os.path.join(dep_install_path, 'lib32')):
+        if is_32bit() and os.path.exists(os.path.join(install_path, 'lib32')):
             lib_dir = 'lib32'
 
-        self.library_dirs.insert(0, os.path.join(dep_install_path, lib_dir))
+        self.library_dirs.insert(0, os.path.join(install_path, lib_dir))
+
+    def run(self):
+        if using_system_libs():
+            print("Skip building dependencies, using system libs.")
+        elif not os.path.exists(os.path.join(PROJECT_DIR, 'crt', 'aws-c-common', 'CMakeLists.txt')):
+            print("Skip building dependencies, source not found.")
+        else:
+            self._build_dependencies()
 
         # continue with normal build_ext.run()
         super().run()

source/checksums.h

@@ -8,5 +8,6 @@
 
 PyObject *aws_py_checksums_crc32(PyObject *self, PyObject *args);
 PyObject *aws_py_checksums_crc32c(PyObject *self, PyObject *args);
+PyObject *aws_py_checksums_crc64nvme(PyObject *self, PyObject *args);
 
 #endif /* AWS_CRT_PYTHON_CHECKSUMS_H */

source/crc.c

@@ -7,7 +7,7 @@
 
 #include "aws/checksums/crc.h"
 #include "aws/common/byte_buf.h"
-PyObject *checksums_crc_common(PyObject *args, uint32_t (*checksum_fn)(const uint8_t *, int, uint32_t)) {
+PyObject *checksums_crc32_common(PyObject *args, uint32_t (*checksum_fn)(const uint8_t *, size_t, uint32_t)) {
     Py_buffer input;
     PyObject *py_previousCrc;
     PyObject *py_result = NULL;
@@ -39,18 +39,11 @@ PyObject *checksums_crc_common(PyObject *args, uint32_t (*checksum_fn)(const uin
 
         /* clang-format off */
         Py_BEGIN_ALLOW_THREADS
-            /* Avoid truncation of length for very large buffers. crc() takes
-               length as an int, which may be narrower than Py_ssize_t. */
-            while ((size_t)len > INT_MAX) {
-                val = checksum_fn(buf, INT_MAX, val);
-                buf += (size_t)INT_MAX;
-                len -= (size_t)INT_MAX;
-            }
-            val = checksum_fn(buf, (int)len, val);
+            val = checksum_fn(buf, (size_t)len, val);
         Py_END_ALLOW_THREADS
         /* clang-format on */
     } else {
-        val = checksum_fn(input.buf, (int)input.len, val);
+        val = checksum_fn(input.buf, (size_t)input.len, val);
     }
     py_result = PyLong_FromUnsignedLong(val);
 done:
@@ -62,10 +55,52 @@ PyObject *checksums_crc_common(PyObject *args, uint32_t (*checksum_fn)(const uin
 
 PyObject *aws_py_checksums_crc32(PyObject *self, PyObject *args) {
     (void)self;
-    return checksums_crc_common(args, aws_checksums_crc32);
+    return checksums_crc32_common(args, aws_checksums_crc32_ex);
 }
 
 PyObject *aws_py_checksums_crc32c(PyObject *self, PyObject *args) {
     (void)self;
-    return checksums_crc_common(args, aws_checksums_crc32c);
+    return checksums_crc32_common(args, aws_checksums_crc32c_ex);
+}
+
+PyObject *aws_py_checksums_crc64nvme(PyObject *self, PyObject *args) {
+    (void)self;
+    Py_buffer input;
+    PyObject *py_previousCrc64;
+    PyObject *py_result = NULL;
+
+    if (!PyArg_ParseTuple(args, "s*O", &input, &py_previousCrc64)) {
+        return NULL;
+    }
+
+    /* Note: PyArg_ParseTuple() doesn't do overflow checking on unsigned values
+     * so use PyLong_AsUnsignedLongLong() to get the value of the previousCrc arg */
+    uint64_t previousCrc = PyLong_AsUnsignedLongLong(py_previousCrc64);
+
+    if (previousCrc == (uint64_t)-1 && PyErr_Occurred()) {
+        goto done;
+    }
+
+    if (!PyBuffer_IsContiguous(&input, 'C')) {
+        PyErr_SetString(PyExc_ValueError, "input must be contiguous buffer");
+        goto done;
+    }
+
+    /* Releasing the GIL for very small buffers is inefficient
+       and may lower performance */
+    if (input.len > 1024 * 5) {
+        /* clang-format off */
+        Py_BEGIN_ALLOW_THREADS
+            previousCrc = aws_checksums_crc64nvme_ex(input.buf, (size_t)input.len, previousCrc);
+        Py_END_ALLOW_THREADS
+        /* clang-format on */
+    } else {
+        previousCrc = aws_checksums_crc64nvme_ex(input.buf, (size_t)input.len, previousCrc);
+    }
+    py_result = PyLong_FromUnsignedLongLong(previousCrc);
+done:
+    if (input.obj) {
+        PyBuffer_Release(&input);
+    }
+    return py_result;
 }

source/module.c

@@ -730,6 +730,7 @@ static PyMethodDef s_module_methods[] = {
     /* Checksum primitives */
     AWS_PY_METHOD_DEF(checksums_crc32, METH_VARARGS),
     AWS_PY_METHOD_DEF(checksums_crc32c, METH_VARARGS),
+    AWS_PY_METHOD_DEF(checksums_crc64nvme, METH_VARARGS),
 
     /* HTTP */
     AWS_PY_METHOD_DEF(http_connection_close, METH_VARARGS),

source/s3_meta_request.c

@@ -24,12 +24,6 @@ struct s3_meta_request_binding {
     /* Reference to python object that reference to other related python object to keep it alive */
     PyObject *py_core;
 
-    /**
-     * file path if set, it handles file operation from C land to reduce the cost
-     * passing chunks from C into python. One for recv/writing, the other for send/reading
-     **/
-    FILE *recv_file;
-
     /* Batch up the transferred size in one sec. */
     uint64_t size_transferred;
     /* The time stamp when the progress reported */
@@ -42,9 +36,6 @@ struct aws_s3_meta_request *aws_py_get_s3_meta_request(PyObject *meta_request) {
 }
 
 static void s_destroy(struct s3_meta_request_binding *meta_request) {
-    if (meta_request->recv_file) {
-        fclose(meta_request->recv_file);
-    }
     Py_XDECREF(meta_request->py_core);
     aws_mem_release(aws_py_get_allocator(), meta_request);
 }
@@ -148,22 +139,6 @@ static int s_s3_request_on_body(
     void *user_data) {
     (void)meta_request;
     struct s3_meta_request_binding *request_binding = user_data;
-
-    if (request_binding->recv_file) {
-        /* The callback will be invoked with the right order, so we don't need to seek first. */
-        if (fwrite((void *)body->ptr, body->len, 1, request_binding->recv_file) < 1) {
-            int errno_value = ferror(request_binding->recv_file) ? errno : 0; /* Always cache errno  */
-            aws_translate_and_raise_io_error_or(errno_value, AWS_ERROR_FILE_WRITE_FAILURE);
-            AWS_LOGF_ERROR(
-                AWS_LS_S3_META_REQUEST,
-                "id=%p Failed writing to file. errno:%d. aws-error:%s",
-                (void *)meta_request,
-                errno_value,
-                aws_error_name(aws_last_error()));
-            return AWS_OP_ERR;
-        }
-        return AWS_OP_SUCCESS;
-    }
     bool error = true;
     /*************** GIL ACQUIRE ***************/
     PyGILState_STATE state;
@@ -201,25 +176,6 @@ static void s_s3_request_on_finish(
     struct s3_meta_request_binding *request_binding = user_data;
 
     int error_code = meta_request_result->error_code;
-
-    if (request_binding->recv_file) {
-        if (fclose(request_binding->recv_file) != 0) {
-            /* Failed to close file, so we can't guarantee it flushed to disk.
-             * If the meta-request's error_code was 0, change it to failure */
-            if (error_code == 0) {
-                int errno_value = errno; /* Always cache errno before potential side-effect */
-                aws_translate_and_raise_io_error_or(errno_value, AWS_ERROR_FILE_WRITE_FAILURE);
-                error_code = aws_last_error();
-                AWS_LOGF_ERROR(
-                    AWS_LS_S3_META_REQUEST,
-                    "id=%p Failed closing file. errno:%d. aws-error:%s",
-                    (void *)meta_request,
-                    errno_value,
-                    aws_error_name(error_code));
-            }
-        }
-        request_binding->recv_file = NULL;
-    }
     /*************** GIL ACQUIRE ***************/
     PyGILState_STATE state;
     if (aws_py_gilstate_ensure(&state)) {
@@ -455,22 +411,14 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
     meta_request->py_core = py_core;
     Py_INCREF(meta_request->py_core);
 
-    if (recv_filepath) {
-        meta_request->recv_file = aws_fopen(recv_filepath, "wb");
-        if (!meta_request->recv_file) {
-            aws_translate_and_raise_io_error(errno);
-            PyErr_SetAwsLastError();
-            goto error;
-        }
-    }
-
     struct aws_s3_meta_request_options s3_meta_request_opt = {
         .type = type,
         .operation_name = aws_byte_cursor_from_c_str(operation_name),
         .message = http_request,
         .signing_config = signing_config,
         .checksum_config = &checksum_config,
         .send_filepath = aws_byte_cursor_from_c_str(send_filepath),
+        .recv_filepath = aws_byte_cursor_from_c_str(recv_filepath),
         .headers_callback = s_s3_request_on_headers,
         .body_callback = s_s3_request_on_body,
         .finish_callback = s_s3_request_on_finish,

test/test_checksums.py

@@ -95,6 +95,48 @@ def test_crc32c_huge_buffer(self):
         val = checksums.crc32c(huge_buffer)
         self.assertEqual(0x572a7c8a, val)
 
+    def test_crc64nvme_zeros_one_shot(self):
+        output = checksums.crc64nvme(bytes(32))
+        expected = 0xcf3473434d4ecf3b
+        self.assertEqual(expected, output)
+
+    def test_crc64nvme_zeros_iterated(self):
+        output = 0
+        for i in range(32):
+            output = checksums.crc64nvme(bytes(1), output)
+        expected = 0xcf3473434d4ecf3b
+        self.assertEqual(expected, output)
+
+    def test_crc64nvme_values_one_shot(self):
+        output = checksums.crc64nvme(''.join(chr(i) for i in range(32)))
+        expected = 0xb9d9d4a8492cbd7f
+        self.assertEqual(expected, output)
+
+    def test_crc64nvme_values_iterated(self):
+        output = 0
+        for i in range(32):
+            output = checksums.crc64nvme(chr(i), output)
+        expected = 0xb9d9d4a8492cbd7f
+        self.assertEqual(expected, output)
+
+    def test_crc64nvme_large_buffer(self):
+        # stress test gil optimization for 32 bit architecture which cannot handle huge buffer
+        large_buffer = bytes(25 * 2**20)
+        val = checksums.crc64nvme(large_buffer)
+        self.assertEqual(0x5b6f5045463ca45e, val)
+
+    def test_crc64nvme_huge_buffer(self):
+        if sys.platform.startswith('freebsd'):
+            # Skip this test for freebsd, as it simply crashes instead of raising exception in this case
+            raise unittest.SkipTest('Skip this test for freebsd')
+        try:
+            INT_MAX = 2**32 - 1
+            huge_buffer = bytes(INT_MAX + 5)
+        except BaseException:
+            raise unittest.SkipTest('Machine cant allocate giant buffer for giant buffer test')
+        val = checksums.crc64nvme(huge_buffer)
+        self.assertEqual(0x2645c28052b1fbb0, val)
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_websocket.py

@@ -467,7 +467,6 @@ def test_send_receive_data(self):
             self.assertIsNone(handler.exception)
 
     def test_send_frame_exceptions(self):
-        init_logging(LogLevel.Trace, 'stderr')
         with WebSocketServer(self.host, self.port) as server:
             handler = ClientHandler()
             handler.connect_sync(self.host, self.port)