From 09a289eba63ddf07dc6faafcef903fa7f9bf7e85 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 3 Sep 2015 15:21:53 -0700 Subject: [PATCH 1/2] upload_file() and download_file() for Bucket and Object Rely on the already-injected meta.client rather then S3Transfer. Unit test, function test, integration test and CHANGELOG are also updated. --- CHANGELOG.rst | 2 ++ boto3/s3/inject.py | 43 +++++++++++++++++++++++++++++++++++- boto3/session.py | 6 ++++- tests/functional/test_s3.py | 14 ++++++++++++ tests/integration/test_s3.py | 22 ++++++++++++++++++ tests/unit/s3/test_inject.py | 36 ++++++++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa74844268..2ae76b1fc5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,8 @@ Next Release - (TBD) * bugfix:Identifier: Make resource identifiers immutable. (`issue 246 `__) +* feature: Both S3 Bucket and Object obtain upload_file() and download_file() + (`issue 243 `__) 1.1.3 - 2015-09-03 diff --git a/boto3/s3/inject.py b/boto3/s3/inject.py index a7988cd118..0793eaa203 100644 --- a/boto3/s3/inject.py +++ b/boto3/s3/inject.py @@ -21,8 +21,17 @@ def inject_s3_transfer_methods(class_attributes, **kwargs): utils.inject_attribute(class_attributes, 'download_file', download_file) -def inject_bucket_load(class_attributes, **kwargs): +def inject_bucket_methods(class_attributes, **kwargs): utils.inject_attribute(class_attributes, 'load', bucket_load) + utils.inject_attribute(class_attributes, 'upload_file', bucket_upload_file) + utils.inject_attribute( + class_attributes, 'download_file', bucket_download_file) + + +def inject_object_methods(class_attributes, **kwargs): + utils.inject_attribute(class_attributes, 'upload_file', object_upload_file) + utils.inject_attribute( + class_attributes, 'download_file', object_download_file) def bucket_load(self, *args, **kwargs): @@ -56,3 +65,35 @@ def download_file(self, Bucket, Key, Filename, ExtraArgs=None, return transfer.download_file( bucket=Bucket, key=Key, filename=Filename, extra_args=ExtraArgs, callback=Callback) + + +def bucket_upload_file(self, Filename, Key, + ExtraArgs=None, Callback=None, Config=None): + """Upload a file to an S3 object.""" + return self.meta.client.upload_file( + Filename=Filename, Bucket=self.name, Key=Key, + ExtraArgs=ExtraArgs, Callback=Callback, Config=Config) + + +def bucket_download_file(self, Key, Filename, + ExtraArgs=None, Callback=None, Config=None): + """Download an S3 object to a file.""" + return self.meta.client.download_file( + Bucket=self.name, Key=Key, Filename=Filename, + ExtraArgs=ExtraArgs, Callback=Callback, Config=Config) + + +def object_upload_file(self, Filename, + ExtraArgs=None, Callback=None, Config=None): + """Upload a file to an S3 object.""" + return self.meta.client.upload_file( + Filename=Filename, Bucket=self.bucket_name, Key=self.key, + ExtraArgs=ExtraArgs, Callback=Callback, Config=Config) + + +def object_download_file(self, Filename, + ExtraArgs=None, Callback=None, Config=None): + """Download an S3 object to a file.""" + return self.meta.client.download_file( + Bucket=self.bucket_name, Key=self.key, Filename=Filename, + ExtraArgs=ExtraArgs, Callback=Callback, Config=Config) diff --git a/boto3/session.py b/boto3/session.py index e35ad9ab93..4e9b5ee37d 100644 --- a/boto3/session.py +++ b/boto3/session.py @@ -291,7 +291,11 @@ def _register_default_handlers(self): self._session.register( 'creating-resource-class.s3.Bucket', boto3.utils.lazy_call( - 'boto3.s3.inject.inject_bucket_load')) + 'boto3.s3.inject.inject_bucket_methods')) + self._session.register( + 'creating-resource-class.s3.Object', + boto3.utils.lazy_call( + 'boto3.s3.inject.inject_object_methods')) # DynamoDb customizations self._session.register( diff --git a/tests/functional/test_s3.py b/tests/functional/test_s3.py index 081a48e49f..31c565af92 100644 --- a/tests/functional/test_s3.py +++ b/tests/functional/test_s3.py @@ -29,3 +29,17 @@ def test_bucket_resource_has_load_method(self): bucket = session.resource('s3').Bucket('fakebucket') self.assertTrue(hasattr(bucket, 'load'), 'load() was not injected onto S3 Bucket resource.') + + def test_transfer_methods_injected_to_bucket(self): + bucket = boto3.resource('s3').Bucket('my_bucket') + self.assertTrue(hasattr(bucket, 'upload_file'), + 'upload_file was not injected onto S3 bucket') + self.assertTrue(hasattr(bucket, 'download_file'), + 'download_file was not injected onto S3 bucket') + + def test_transfer_methods_injected_to_object(self): + obj = boto3.resource('s3').Object('my_bucket', 'my_key') + self.assertTrue(hasattr(obj, 'upload_file'), + 'upload_file was not injected onto S3 object') + self.assertTrue(hasattr(obj, 'download_file'), + 'download_file was not injected onto S3 object') diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py index 4f4ee767b2..d647d646c2 100644 --- a/tests/integration/test_s3.py +++ b/tests/integration/test_s3.py @@ -483,6 +483,28 @@ def test_transfer_methods_through_client(self): Filename=download_path) assert_files_equal(filename, download_path) + def test_transfer_methods_through_bucket(self): + # This is just a sanity check to ensure that the bucket interface work. + key = 'bucket.txt' + bucket = self.session.resource('s3').Bucket(self.bucket_name) + filename = self.files.create_file_with_size(key, 1024*1024) + bucket.upload_file(Filename=filename, Key=key) + self.addCleanup(self.delete_object, key) + download_path = os.path.join(self.files.rootdir, unique_id('foo')) + bucket.download_file(Key=key, Filename=download_path) + assert_files_equal(filename, download_path) + + def test_transfer_methods_through_object(self): + # This is just a sanity check to ensure that the object interface work. + key = 'object.txt' + obj = self.session.resource('s3').Object(self.bucket_name, key) + filename = self.files.create_file_with_size(key, 1024*1024) + obj.upload_file(Filename=filename) + self.addCleanup(self.delete_object, key) + download_path = os.path.join(self.files.rootdir, unique_id('foo')) + obj.download_file(Filename=download_path) + assert_files_equal(filename, download_path) + class TestCustomS3BucketLoad(unittest.TestCase): def setUp(self): diff --git a/tests/unit/s3/test_inject.py b/tests/unit/s3/test_inject.py index 6f277d6632..666a23819b 100644 --- a/tests/unit/s3/test_inject.py +++ b/tests/unit/s3/test_inject.py @@ -75,3 +75,39 @@ def test_bucket_load_raise_error(self): } with self.assertRaises(ClientError): inject.bucket_load(self.resource) + + +class TestBucketTransferMethods(unittest.TestCase): + + def setUp(self): + self.bucket = mock.Mock(name='my_bucket') + + def test_upload_file_proxies_to_meta_client(self): + inject.bucket_upload_file(self.bucket, Filename='foo', Key='key') + self.bucket.meta.client.upload_file.assert_called_with( + Filename='foo', Bucket=self.bucket.name, Key='key', + ExtraArgs=None, Callback=None, Config=None) + + def test_download_file_proxies_to_meta_client(self): + inject.bucket_download_file(self.bucket, Key='key', Filename='foo') + self.bucket.meta.client.download_file.assert_called_with( + Bucket=self.bucket.name, Key='key', Filename='foo', + ExtraArgs=None, Callback=None, Config=None) + + +class TestObjectTransferMethods(unittest.TestCase): + + def setUp(self): + self.obj = mock.Mock(bucket_name='my_bucket', key='my_key') + + def test_upload_file_proxies_to_meta_client(self): + inject.object_upload_file(self.obj, Filename='foo') + self.obj.meta.client.upload_file.assert_called_with( + Filename='foo', Bucket=self.obj.bucket_name, Key=self.obj.key, + ExtraArgs=None, Callback=None, Config=None) + + def test_download_file_proxies_to_meta_client(self): + inject.object_download_file(self.obj, Filename='foo') + self.obj.meta.client.download_file.assert_called_with( + Bucket=self.obj.bucket_name, Key=self.obj.key, Filename='foo', + ExtraArgs=None, Callback=None, Config=None) From a59b6a09ee9fdcac001cf9e0b13264ef0bde019e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 19 Sep 2015 13:03:21 -0700 Subject: [PATCH 2/2] Document all 8 upload_file() and download_file() methods --- boto3/s3/inject.py | 76 +++++++++++++++++++++++++++++++++++++++++--- boto3/s3/transfer.py | 16 +++++++--- 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/boto3/s3/inject.py b/boto3/s3/inject.py index 0793eaa203..02a7a68df1 100644 --- a/boto3/s3/inject.py +++ b/boto3/s3/inject.py @@ -53,6 +53,18 @@ def bucket_load(self, *args, **kwargs): def upload_file(self, Filename, Bucket, Key, ExtraArgs=None, Callback=None, Config=None): + """Upload a file to an S3 object. + + Usage:: + + import boto3 + s3 = boto3.resource('s3') + s3.meta.client.upload_file('/tmp/hello.txt', 'mybucket', 'hello.txt') + + Similar behavior as S3Transfer's upload_file() method, + except that parameters are capitalized. Detailed examples can be found at + :ref:`S3Transfer's Usage `. + """ transfer = S3Transfer(self, Config) return transfer.upload_file( filename=Filename, bucket=Bucket, key=Key, @@ -61,6 +73,18 @@ def upload_file(self, Filename, Bucket, Key, ExtraArgs=None, def download_file(self, Bucket, Key, Filename, ExtraArgs=None, Callback=None, Config=None): + """Download an S3 object to a file. + + Usage:: + + import boto3 + s3 = boto3.resource('s3') + s3.meta.client.download_file('mybucket', 'hello.txt', '/tmp/hello.txt') + + Similar behavior as S3Transfer's download_file() method, + except that parameters are capitalized. Detailed examples can be found at + :ref:`S3Transfer's Usage `. + """ transfer = S3Transfer(self, Config) return transfer.download_file( bucket=Bucket, key=Key, filename=Filename, @@ -69,7 +93,18 @@ def download_file(self, Bucket, Key, Filename, ExtraArgs=None, def bucket_upload_file(self, Filename, Key, ExtraArgs=None, Callback=None, Config=None): - """Upload a file to an S3 object.""" + """Upload a file to an S3 object. + + Usage:: + + import boto3 + s3 = boto3.resource('s3') + s3.Bucket('mybucket').upload_file('/tmp/hello.txt', 'hello.txt') + + Similar behavior as S3Transfer's upload_file() method, + except that parameters are capitalized. Detailed examples can be found at + :ref:`S3Transfer's Usage `. + """ return self.meta.client.upload_file( Filename=Filename, Bucket=self.name, Key=Key, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config) @@ -77,7 +112,18 @@ def bucket_upload_file(self, Filename, Key, def bucket_download_file(self, Key, Filename, ExtraArgs=None, Callback=None, Config=None): - """Download an S3 object to a file.""" + """Download an S3 object to a file. + + Usage:: + + import boto3 + s3 = boto3.resource('s3') + s3.Bucket('mybucket').download_file('hello.txt', '/tmp/hello.txt') + + Similar behavior as S3Transfer's download_file() method, + except that parameters are capitalized. Detailed examples can be found at + :ref:`S3Transfer's Usage `. + """ return self.meta.client.download_file( Bucket=self.name, Key=Key, Filename=Filename, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config) @@ -85,7 +131,18 @@ def bucket_download_file(self, Key, Filename, def object_upload_file(self, Filename, ExtraArgs=None, Callback=None, Config=None): - """Upload a file to an S3 object.""" + """Upload a file to an S3 object. + + Usage:: + + import boto3 + s3 = boto3.resource('s3') + s3.Object('mybucket', 'hello.txt').upload_file('/tmp/hello.txt') + + Similar behavior as S3Transfer's upload_file() method, + except that parameters are capitalized. Detailed examples can be found at + :ref:`S3Transfer's Usage `. + """ return self.meta.client.upload_file( Filename=Filename, Bucket=self.bucket_name, Key=self.key, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config) @@ -93,7 +150,18 @@ def object_upload_file(self, Filename, def object_download_file(self, Filename, ExtraArgs=None, Callback=None, Config=None): - """Download an S3 object to a file.""" + """Download an S3 object to a file. + + Usage:: + + import boto3 + s3 = boto3.resource('s3') + s3.Object('mybucket', 'hello.txt').download_file('/tmp/hello.txt') + + Similar behavior as S3Transfer's download_file() method, + except that parameters are capitalized. Detailed examples can be found at + :ref:`S3Transfer's Usage `. + """ return self.meta.client.download_file( Bucket=self.bucket_name, Key=self.key, Filename=Filename, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config) diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 15f85d9cf3..6ec5ad9e6b 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -38,6 +38,8 @@ time. +.. _ref_s3transfer_usage: + Usage ===== @@ -606,6 +608,11 @@ def __init__(self, client, config=None, osutil=None): def upload_file(self, filename, bucket, key, callback=None, extra_args=None): + """Upload a file to an S3 object. + + Variants have also been injected into S3 client, Bucket and Object. + You don't have to use S3Transfer.upload_file() directly. + """ if extra_args is None: extra_args = {} self._validate_all_known_args(extra_args, self.ALLOWED_UPLOAD_ARGS) @@ -636,11 +643,12 @@ def download_file(self, bucket, key, filename, extra_args=None, callback=None): """Download an S3 object to a file. - This method will issue a ``head_object`` request to determine - the size of the S3 object. This is used to determine if the - object is downloaded in parallel. - + Variants have also been injected into S3 client, Bucket and Object. + You don't have to use S3Transfer.download_file() directly. """ + # This method will issue a ``head_object`` request to determine + # the size of the S3 object. This is used to determine if the + # object is downloaded in parallel. if extra_args is None: extra_args = {} self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS)