From 8f2ce89e6ccd5cf4af06491af6c2154e7c3bd3c6 Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Tue, 14 Apr 2020 20:10:38 +0300 Subject: [PATCH 01/11] added overwrite option --- storages/backends/dropbox.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index b2a0c1616..be145efbc 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -21,11 +21,12 @@ from django.utils.deconstruct import deconstructible from dropbox import Dropbox from dropbox.exceptions import ApiError -from dropbox.files import CommitInfo, FolderMetadata, UploadSessionCursor +from dropbox.files import CommitInfo, FolderMetadata, UploadSessionCursor, WriteMode from storages.utils import setting _DEFAULT_TIMEOUT = 100 +_DEFAULT_OVERWRITE = False class DropBoxStorageException(Exception): @@ -69,15 +70,18 @@ class DropBoxStorage(Storage): location = setting('DROPBOX_ROOT_PATH', '/') oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN') timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) + overwrite = setting('DROPBOX_OVERWRITE', _DEFAULT_OVERWRITE) CHUNK_SIZE = 4 * 1024 * 1024 - def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout): + def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout, + overwrite=overwrite): if oauth2_access_token is None: raise ImproperlyConfigured("You must configure an auth token at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.root_path = root_path + self.overwrite = overwrite self.client = Dropbox(oauth2_access_token, timeout=timeout) def _full_path(self, name): @@ -89,6 +93,9 @@ def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): + if self.overwrite: + return False + try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: @@ -132,7 +139,7 @@ def _open(self, name, mode='rb'): def _save(self, name, content): content.open() if content.size <= self.CHUNK_SIZE: - self.client.files_upload(content.read(), self._full_path(name)) + self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode('overwrite')) else: self._chunked_upload(content, self._full_path(name)) content.close() @@ -146,7 +153,7 @@ def _chunked_upload(self, content, dest_path): session_id=upload_session.session_id, offset=content.tell() ) - commit = CommitInfo(path=dest_path) + commit = CommitInfo(path=dest_path, mode=WriteMode('overwrite')) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: From 9a7fb7b75e356d0bd2bcd50fafadca7650d30fe2 Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Tue, 14 Apr 2020 20:10:54 +0300 Subject: [PATCH 02/11] added myself as AUTHOR --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 4fc69fc3d..aedcf25ac 100644 --- a/AUTHORS +++ b/AUTHORS @@ -42,6 +42,7 @@ By order of apparition, thanks: * Shaung Cheng (S3 docs) * Andrew Perry (Bug fixes in SFTPStorage) * Manuel Kaufmann (humitos) + * Taras Petriichuk (Dropbox autorename option) Extra thanks to Marty for adding this in Django, From 6d60386136fa91172af88ef3e4796c08c9200fed Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 00:49:26 +0300 Subject: [PATCH 03/11] Refactored to native feature implementation after comment from package owner --- storages/backends/dropbox.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index be145efbc..47935c290 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -10,6 +10,7 @@ from __future__ import absolute_import +import posixpath from io import BytesIO from shutil import copyfileobj from tempfile import SpooledTemporaryFile @@ -23,10 +24,9 @@ from dropbox.exceptions import ApiError from dropbox.files import CommitInfo, FolderMetadata, UploadSessionCursor, WriteMode -from storages.utils import setting +from storages.utils import setting, get_available_overwrite_name _DEFAULT_TIMEOUT = 100 -_DEFAULT_OVERWRITE = False class DropBoxStorageException(Exception): @@ -70,18 +70,18 @@ class DropBoxStorage(Storage): location = setting('DROPBOX_ROOT_PATH', '/') oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN') timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) - overwrite = setting('DROPBOX_OVERWRITE', _DEFAULT_OVERWRITE) + file_overwrite = setting('DROPBOX_FILE_OVERWRITE', True) CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout, - overwrite=overwrite): + file_overwrite=file_overwrite): if oauth2_access_token is None: raise ImproperlyConfigured("You must configure an auth token at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.root_path = root_path - self.overwrite = overwrite + self.overwrite = file_overwrite self.client = Dropbox(oauth2_access_token, timeout=timeout) def _full_path(self, name): @@ -93,9 +93,6 @@ def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): - if self.overwrite: - return False - try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: @@ -165,3 +162,24 @@ def _chunked_upload(self, content, dest_path): content.read(self.CHUNK_SIZE), cursor ) cursor.offset = content.tell() + + def _clean_name(self, name): + """ + Cleans the name so that Windows style paths work + """ + # Normalize Windows style paths + clean_name = posixpath.normpath(name).replace('\\', '/') + + # os.path.normpath() can strip trailing slashes so we implement + # a workaround here. + if name.endswith('/') and not clean_name.endswith('/'): + # Add a trailing slash as it was stripped. + clean_name += '/' + return clean_name + + def get_available_name(self, name, max_length=None): + """Overwrite existing file with the same name.""" + name = self._clean_name(name) + if self.file_overwrite: + return get_available_overwrite_name(name, max_length) + return super(DropBoxStorage, self).get_available_name(name, max_length) From 451805043a4206ee2db8b49f03cfe18e85a53456 Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 01:06:43 +0300 Subject: [PATCH 04/11] Make write mode as external option, with default set to overwrite --- storages/backends/dropbox.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index 47935c290..1f8ab3aa7 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -71,17 +71,19 @@ class DropBoxStorage(Storage): oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN') timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) file_overwrite = setting('DROPBOX_FILE_OVERWRITE', True) + write_mode = setting('DROPBOX_WRITE_MODE', 'overwrite') CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout, - file_overwrite=file_overwrite): + file_overwrite=file_overwrite, write_mode=write_mode): if oauth2_access_token is None: raise ImproperlyConfigured("You must configure an auth token at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.root_path = root_path - self.overwrite = file_overwrite + self.file_overwrite = file_overwrite + self.write_mode = write_mode self.client = Dropbox(oauth2_access_token, timeout=timeout) def _full_path(self, name): @@ -136,7 +138,7 @@ def _open(self, name, mode='rb'): def _save(self, name, content): content.open() if content.size <= self.CHUNK_SIZE: - self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode('overwrite')) + self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode(self.write_mode)) else: self._chunked_upload(content, self._full_path(name)) content.close() @@ -150,7 +152,7 @@ def _chunked_upload(self, content, dest_path): session_id=upload_session.session_id, offset=content.tell() ) - commit = CommitInfo(path=dest_path, mode=WriteMode('overwrite')) + commit = CommitInfo(path=dest_path, mode=WriteMode(self.write_mode)) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: From d148bb7142d2cab926e865c326e4e2d8c80369c2 Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 08:18:06 +0300 Subject: [PATCH 05/11] Fix order of imports --- storages/backends/dropbox.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index 1f8ab3aa7..30cae4f6e 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -24,7 +24,7 @@ from dropbox.exceptions import ApiError from dropbox.files import CommitInfo, FolderMetadata, UploadSessionCursor, WriteMode -from storages.utils import setting, get_available_overwrite_name +from storages.utils import get_available_overwrite_name, setting _DEFAULT_TIMEOUT = 100 @@ -70,7 +70,7 @@ class DropBoxStorage(Storage): location = setting('DROPBOX_ROOT_PATH', '/') oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN') timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) - file_overwrite = setting('DROPBOX_FILE_OVERWRITE', True) + file_overwrite = setting('DROPBOX_FILE_OVERWRITE', False) write_mode = setting('DROPBOX_WRITE_MODE', 'overwrite') CHUNK_SIZE = 4 * 1024 * 1024 From 2b7f8c1d7bc6c70b6e9884e40e8d4cc1158641f6 Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 08:19:24 +0300 Subject: [PATCH 06/11] added new options description --- docs/backends/dropbox.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/backends/dropbox.rst b/docs/backends/dropbox.rst index d4ce6b9f4..60e94011e 100644 --- a/docs/backends/dropbox.rst +++ b/docs/backends/dropbox.rst @@ -26,5 +26,13 @@ To use DropBoxStorage set:: Timeout in seconds for making requests to the API. If ``None``, the client will wait forever. The default is ``100`` seconds which is the current default in the official SDK. +``DROPBOX_FILE_OVERWRITE`` (optional) + Overwrite an existing file when it has the same name as the file being uploaded. + Otherwise, rename it. Default is ``False`` + +``DROPBOX_WRITE_MODE`` (optional) + Allow to set Dropbox WriteMode strategy. + Default is ``overwrite`` + .. _`tutorial`: https://www.dropbox.com/developers/documentation/python#tutorial .. _`Dropbox SDK for Python`: https://www.dropbox.com/developers/documentation/python#tutorial From cd8eaccbb1b9c23f77217a8be1c6a408e5d13909 Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 09:03:06 +0300 Subject: [PATCH 07/11] fixed import sort issue --- storages/backends/dropbox.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index 30cae4f6e..337b68dc3 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -22,7 +22,9 @@ from django.utils.deconstruct import deconstructible from dropbox import Dropbox from dropbox.exceptions import ApiError -from dropbox.files import CommitInfo, FolderMetadata, UploadSessionCursor, WriteMode +from dropbox.files import ( + CommitInfo, FolderMetadata, UploadSessionCursor, WriteMode, +) from storages.utils import get_available_overwrite_name, setting From b44e774571edaa8d1eceeb45a60e9cc6d0ff487a Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 09:36:36 +0300 Subject: [PATCH 08/11] removed redundand option, updated documentation --- docs/backends/dropbox.rst | 6 +----- storages/backends/dropbox.py | 9 ++++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/backends/dropbox.rst b/docs/backends/dropbox.rst index 60e94011e..f01db3cd1 100644 --- a/docs/backends/dropbox.rst +++ b/docs/backends/dropbox.rst @@ -26,13 +26,9 @@ To use DropBoxStorage set:: Timeout in seconds for making requests to the API. If ``None``, the client will wait forever. The default is ``100`` seconds which is the current default in the official SDK. -``DROPBOX_FILE_OVERWRITE`` (optional) - Overwrite an existing file when it has the same name as the file being uploaded. - Otherwise, rename it. Default is ``False`` - ``DROPBOX_WRITE_MODE`` (optional) Allow to set Dropbox WriteMode strategy. - Default is ``overwrite`` + Default is ``add``. Read more at https://dropbox-sdk-python.readthedocs.io/en/latest/api/files.html#dropbox.files.CommitInfo.mode .. _`tutorial`: https://www.dropbox.com/developers/documentation/python#tutorial .. _`Dropbox SDK for Python`: https://www.dropbox.com/developers/documentation/python#tutorial diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index 337b68dc3..1b0f3dc36 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -29,6 +29,7 @@ from storages.utils import get_available_overwrite_name, setting _DEFAULT_TIMEOUT = 100 +_DEFAULT_MODE = 'add' class DropBoxStorageException(Exception): @@ -72,19 +73,17 @@ class DropBoxStorage(Storage): location = setting('DROPBOX_ROOT_PATH', '/') oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN') timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) - file_overwrite = setting('DROPBOX_FILE_OVERWRITE', False) - write_mode = setting('DROPBOX_WRITE_MODE', 'overwrite') + write_mode = setting('DROPBOX_WRITE_MODE', _DEFAULT_MODE) CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout, - file_overwrite=file_overwrite, write_mode=write_mode): + write_mode=write_mode): if oauth2_access_token is None: raise ImproperlyConfigured("You must configure an auth token at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.root_path = root_path - self.file_overwrite = file_overwrite self.write_mode = write_mode self.client = Dropbox(oauth2_access_token, timeout=timeout) @@ -184,6 +183,6 @@ def _clean_name(self, name): def get_available_name(self, name, max_length=None): """Overwrite existing file with the same name.""" name = self._clean_name(name) - if self.file_overwrite: + if self.write_mode == 'overwrite': return get_available_overwrite_name(name, max_length) return super(DropBoxStorage, self).get_available_name(name, max_length) From 8aa6dfe36b2f57342ef8a2bbaa0f9512aa3ac28f Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 09:40:07 +0300 Subject: [PATCH 09/11] replaced _clean_name by _full_path --- storages/backends/dropbox.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index 1b0f3dc36..28848e509 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -166,23 +166,9 @@ def _chunked_upload(self, content, dest_path): ) cursor.offset = content.tell() - def _clean_name(self, name): - """ - Cleans the name so that Windows style paths work - """ - # Normalize Windows style paths - clean_name = posixpath.normpath(name).replace('\\', '/') - - # os.path.normpath() can strip trailing slashes so we implement - # a workaround here. - if name.endswith('/') and not clean_name.endswith('/'): - # Add a trailing slash as it was stripped. - clean_name += '/' - return clean_name - def get_available_name(self, name, max_length=None): """Overwrite existing file with the same name.""" - name = self._clean_name(name) + name = self._full_path(name) if self.write_mode == 'overwrite': return get_available_overwrite_name(name, max_length) return super(DropBoxStorage, self).get_available_name(name, max_length) From 053298961ea07893c4db70826393e5a77817e7b8 Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 09:43:43 +0300 Subject: [PATCH 10/11] update author change comment --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 5816caef8..3581f0b40 100644 --- a/AUTHORS +++ b/AUTHORS @@ -42,7 +42,7 @@ By order of apparition, thanks: * Shaung Cheng (S3 docs) * Andrew Perry (Bug fixes in SFTPStorage) * Manuel Kaufmann (humitos) - * Taras Petriichuk (Dropbox overwrite option) + * Taras Petriichuk (Dropbox write_mode option) * Zoe Liao (S3 docs) From 92e35d18551c7f919316bde0a9f7884e4eaeff4c Mon Sep 17 00:00:00 2001 From: Taras Petriichuk Date: Thu, 16 Apr 2020 09:46:17 +0300 Subject: [PATCH 11/11] removed unused import posixpath --- storages/backends/dropbox.py | 1 - 1 file changed, 1 deletion(-) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index 28848e509..815bc122a 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -10,7 +10,6 @@ from __future__ import absolute_import -import posixpath from io import BytesIO from shutil import copyfileobj from tempfile import SpooledTemporaryFile