diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 58635976..2faa4ae2 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v2 - name: Setup up Python 3.11 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.11" @@ -26,21 +26,21 @@ jobs: strategy: matrix: include: - - {python: '3.8', os: ubuntu-20.04} - - {python: '3.9', os: ubuntu-20.04} - - {python: '3.10', os: ubuntu-20.04} - - {python: '3.11', os: ubuntu-20.04} - - {python: '3.12', os: ubuntu-20.04} - - - {python: '3.8', os: windows-2019} - - {python: '3.9', os: windows-2019} - - {python: '3.10', os: windows-2019} - - {python: '3.11', os: windows-2019} - - {python: '3.12', os: windows-2019} + - {python-version: '3.8', os: ubuntu-20.04} + - {python-version: '3.9', os: ubuntu-20.04} + - {python-version: '3.10', os: ubuntu-20.04} + - {python-version: '3.11', os: ubuntu-20.04} + - {python-version: '3.12', os: ubuntu-20.04} + + - {python-version: '3.8', os: windows-2019} + - {python-version: '3.9', os: windows-2019} + - {python-version: '3.10', os: windows-2019} + - {python-version: '3.11', os: windows-2019} + - {python-version: '3.12', os: windows-2019} steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -71,25 +71,25 @@ jobs: strategy: matrix: include: - - {python: '3.8', os: ubuntu-20.04} - - {python: '3.9', os: ubuntu-20.04} - - {python: '3.10', os: ubuntu-20.04} - - {python: '3.11', os: ubuntu-20.04} - - {python: '3.12', os: ubuntu-20.04} + - {python-version: '3.8', os: ubuntu-20.04} + - {python-version: '3.9', os: ubuntu-20.04} + - {python-version: '3.10', os: ubuntu-20.04} + - {python-version: '3.11', os: ubuntu-20.04} + - {python-version: '3.12', os: ubuntu-20.04} # # Some of the doctests don't pass on Windows because of Windows-specific # character encoding issues. # - # - {python: '3.7', os: windows-2019} - # - {python: '3.8', os: windows-2019} - # - {python: '3.9', os: windows-2019} - # - {python: '3.10', os: windows-2019} + # - {pythonversion: '3.7', os: windows-2019} + # - {pythonversion: '3.8', os: windows-2019} + # - {pythonversion: '3.9', os: windows-2019} + # - {pythonversion: '3.10', os: windows-2019} steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -114,23 +114,23 @@ jobs: strategy: matrix: include: - - {python: '3.8', os: ubuntu-20.04} - - {python: '3.9', os: ubuntu-20.04} - - {python: '3.10', os: ubuntu-20.04} - - {python: '3.11', os: ubuntu-20.04} - - {python: '3.12', os: ubuntu-20.04} + - {python-version: '3.8', os: ubuntu-20.04} + - {python-version: '3.9', os: ubuntu-20.04} + - {python-version: '3.10', os: ubuntu-20.04} + - {python-version: '3.11', os: ubuntu-20.04} + - {python-version: '3.12', os: ubuntu-20.04} # Not sure why we exclude these, perhaps for historical reasons? # - # - {python: '3.7', os: windows-2019} - # - {python: '3.8', os: windows-2019} - # - {python: '3.9', os: windows-2019} - # - {python: '3.10', os: windows-2019} + # - {python-version: '3.7', os: windows-2019} + # - {python-version: '3.8', os: windows-2019} + # - {python-version: '3.9', os: windows-2019} + # - {python-version: '3.10', os: windows-2019} steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -148,7 +148,8 @@ jobs: - run: bash ci_helpers/helpers.sh enable_moto_server if: ${{ matrix.moto_server }} - - run: | + - name: Start vsftpd + run: | sudo apt-get install vsftpd sudo bash ci_helpers/helpers.sh create_ftp_ftps_servers @@ -169,21 +170,21 @@ jobs: strategy: matrix: include: - - {python: '3.8', os: ubuntu-20.04} - - {python: '3.9', os: ubuntu-20.04} - - {python: '3.10', os: ubuntu-20.04} - - {python: '3.11', os: ubuntu-20.04} - - {python: '3.12', os: ubuntu-20.04} + - {python-version: '3.8', os: ubuntu-20.04} + - {python-version: '3.9', os: ubuntu-20.04} + - {python-version: '3.10', os: ubuntu-20.04} + - {python-version: '3.11', os: ubuntu-20.04} + - {python-version: '3.12', os: ubuntu-20.04} - # - {python: '3.7', os: windows-2019} - # - {python: '3.8', os: windows-2019} - # - {python: '3.9', os: windows-2019} - # - {python: '3.10', os: windows-2019} + # - {python-version: '3.7', os: windows-2019} + # - {python-version: '3.8', os: windows-2019} + # - {python-version: '3.9', os: windows-2019} + # - {python-version: '3.10', os: windows-2019} steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/ci_helpers/helpers.sh b/ci_helpers/helpers.sh index 4533b337..e0f2ece0 100644 --- a/ci_helpers/helpers.sh +++ b/ci_helpers/helpers.sh @@ -20,6 +20,8 @@ create_ftp_ftps_servers(){ mkdir $home_dir useradd -p $(echo $pass | openssl passwd -1 -stdin) -d $home_dir $user chown $user:$user $home_dir + openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout /etc/vsftpd.pem -out /etc/vsftpd.pem -subj "/C=PE/ST=Lima/L=Lima/O=Acme Inc. /OU=IT Department/CN=localhost" + chmod 755 /etc/vsftpd.pem server_setup=''' listen=YES @@ -32,6 +34,7 @@ chroot_local_user=YES allow_writeable_chroot=YES''' additional_ssl_setup=''' +rsa_cert_file=/etc/vsftpd.pem ssl_enable=YES allow_anon_ssl=NO force_local_data_ssl=NO diff --git a/integration-tests/test_ftp.py b/integration-tests/test_ftp.py index 000faae7..94b4e037 100644 --- a/integration-tests/test_ftp.py +++ b/integration-tests/test_ftp.py @@ -1,6 +1,11 @@ from __future__ import unicode_literals import pytest from smart_open import open +import ssl +from functools import partial + +# localhost has self-signed cert, see ci_helpers/helpers.sh:create_ftp_ftps_servers +ssl.create_default_context = partial(ssl.create_default_context, cafile="/etc/vsftpd.pem") @pytest.fixture(params=[("ftp", 21), ("ftps", 90)]) @@ -81,4 +86,4 @@ def test_line_endings_binary(server_info): with open(f"{server_type}://user:123@localhost:{port_num}/file4", "rb") as f: for line in f: - assert B_CLRF in line \ No newline at end of file + assert B_CLRF in line diff --git a/setup.py b/setup.py index a9a4fc53..e2e7cd87 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,6 @@ def read(fname): tests_require = all_deps + [ 'moto[server]', 'responses', - 'boto3', 'pytest', 'pytest-rerunfailures', ] diff --git a/smart_open/ftp.py b/smart_open/ftp.py index 40d54ca3..a7212ecd 100644 --- a/smart_open/ftp.py +++ b/smart_open/ftp.py @@ -86,7 +86,7 @@ def convert_transport_params_to_args(transport_params): def _connect(hostname, username, port, password, secure_connection, transport_params): kwargs = convert_transport_params_to_args(transport_params) if secure_connection: - ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) + ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) ftp = FTP_TLS(context=ssl_context, **kwargs) else: ftp = FTP(**kwargs) diff --git a/smart_open/tests/test_http.py b/smart_open/tests/test_http.py index f4f8338a..9433043f 100644 --- a/smart_open/tests/test_http.py +++ b/smart_open/tests/test_http.py @@ -21,24 +21,22 @@ URL = 'http://localhost' HTTPS_URL = 'https://localhost' HEADERS = { - 'Content-Length': str(len(BYTES)), 'Accept-Ranges': 'bytes', } def request_callback(request, headers=HEADERS, data=BYTES): - try: - range_string = request.headers['range'] - except KeyError: - return (200, headers, data) + headers = headers.copy() + range_string = request.headers.get('range', 'bytes=0-') - start, end = range_string.replace('bytes=', '').split('-', 1) + start, end = range_string.replace('bytes=', '', 1).split('-', 1) start = int(start) - if end: - end = int(end) - else: - end = len(data) - return (200, headers, data[start:end]) + end = int(end) if end else len(data) + + data = data[start:end] + headers['Content-Length'] = str(len(data)) + + return (200, headers, data) @unittest.skipIf(os.environ.get('TRAVIS'), 'This test does not work on TravisCI for some reason')