Merge pull request #243 from rayluo/feature-s3-transfer-methods-for-b…

…ucket-and-object

upload_file() and download_file() for Bucket and Object

CHANGELOG.rst

@@ -6,6 +6,8 @@ Next Release - (TBD)
 
 * bugfix:Identifier: Make resource identifiers immutable.
   (`issue 246 <https://github.com/boto/boto3/pull/246>`__)
+* feature: Both S3 Bucket and Object obtain upload_file() and download_file()
+  (`issue 243 <https://github.com/boto/boto3/pull/243>`__)
 
 
 1.1.3 - 2015-09-03

boto3/s3/inject.py

@@ -21,8 +21,17 @@ def inject_s3_transfer_methods(class_attributes, **kwargs):
     utils.inject_attribute(class_attributes, 'download_file', download_file)
 
 
-def inject_bucket_load(class_attributes, **kwargs):
+def inject_bucket_methods(class_attributes, **kwargs):
     utils.inject_attribute(class_attributes, 'load', bucket_load)
+    utils.inject_attribute(class_attributes, 'upload_file', bucket_upload_file)
+    utils.inject_attribute(
+        class_attributes, 'download_file', bucket_download_file)
+
+
+def inject_object_methods(class_attributes, **kwargs):
+    utils.inject_attribute(class_attributes, 'upload_file', object_upload_file)
+    utils.inject_attribute(
+        class_attributes, 'download_file', object_download_file)
 
 
 def bucket_load(self, *args, **kwargs):
@@ -44,6 +53,18 @@ def bucket_load(self, *args, **kwargs):
 
 def upload_file(self, Filename, Bucket, Key, ExtraArgs=None,
                 Callback=None, Config=None):
+    """Upload a file to an S3 object.
+
+    Usage::
+
+        import boto3
+        s3 = boto3.resource('s3')
+        s3.meta.client.upload_file('/tmp/hello.txt', 'mybucket', 'hello.txt')
+
+    Similar behavior as S3Transfer's upload_file() method,
+    except that parameters are capitalized. Detailed examples can be found at
+    :ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
+    """
     transfer = S3Transfer(self, Config)
     return transfer.upload_file(
         filename=Filename, bucket=Bucket, key=Key,
@@ -52,7 +73,95 @@ def upload_file(self, Filename, Bucket, Key, ExtraArgs=None,
 
 def download_file(self, Bucket, Key, Filename, ExtraArgs=None,
                   Callback=None, Config=None):
+    """Download an S3 object to a file.
+
+    Usage::
+
+        import boto3
+        s3 = boto3.resource('s3')
+        s3.meta.client.download_file('mybucket', 'hello.txt', '/tmp/hello.txt')
+
+    Similar behavior as S3Transfer's download_file() method,
+    except that parameters are capitalized. Detailed examples can be found at
+    :ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
+    """
     transfer = S3Transfer(self, Config)
     return transfer.download_file(
         bucket=Bucket, key=Key, filename=Filename,
         extra_args=ExtraArgs, callback=Callback)
+
+
+def bucket_upload_file(self, Filename, Key,
+                       ExtraArgs=None, Callback=None, Config=None):
+    """Upload a file to an S3 object.
+
+    Usage::
+
+        import boto3
+        s3 = boto3.resource('s3')
+        s3.Bucket('mybucket').upload_file('/tmp/hello.txt', 'hello.txt')
+
+    Similar behavior as S3Transfer's upload_file() method,
+    except that parameters are capitalized. Detailed examples can be found at
+    :ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
+    """
+    return self.meta.client.upload_file(
+        Filename=Filename, Bucket=self.name, Key=Key,
+        ExtraArgs=ExtraArgs, Callback=Callback, Config=Config)
+
+
+def bucket_download_file(self, Key, Filename,
+                         ExtraArgs=None, Callback=None, Config=None):
+    """Download an S3 object to a file.
+
+    Usage::
+
+        import boto3
+        s3 = boto3.resource('s3')
+        s3.Bucket('mybucket').download_file('hello.txt', '/tmp/hello.txt')
+
+    Similar behavior as S3Transfer's download_file() method,
+    except that parameters are capitalized. Detailed examples can be found at
+    :ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
+    """
+    return self.meta.client.download_file(
+        Bucket=self.name, Key=Key, Filename=Filename,
+        ExtraArgs=ExtraArgs, Callback=Callback, Config=Config)
+
+
+def object_upload_file(self, Filename,
+                       ExtraArgs=None, Callback=None, Config=None):
+    """Upload a file to an S3 object.
+
+    Usage::
+
+        import boto3
+        s3 = boto3.resource('s3')
+        s3.Object('mybucket', 'hello.txt').upload_file('/tmp/hello.txt')
+
+    Similar behavior as S3Transfer's upload_file() method,
+    except that parameters are capitalized. Detailed examples can be found at
+    :ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
+    """
+    return self.meta.client.upload_file(
+        Filename=Filename, Bucket=self.bucket_name, Key=self.key,
+        ExtraArgs=ExtraArgs, Callback=Callback, Config=Config)
+
+
+def object_download_file(self, Filename,
+                         ExtraArgs=None, Callback=None, Config=None):
+    """Download an S3 object to a file.
+
+    Usage::
+
+        import boto3
+        s3 = boto3.resource('s3')
+        s3.Object('mybucket', 'hello.txt').download_file('/tmp/hello.txt')
+
+    Similar behavior as S3Transfer's download_file() method,
+    except that parameters are capitalized. Detailed examples can be found at
+    :ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
+    """
+    return self.meta.client.download_file(
+        Bucket=self.bucket_name, Key=self.key, Filename=Filename,
+        ExtraArgs=ExtraArgs, Callback=Callback, Config=Config)

boto3/s3/transfer.py

@@ -38,6 +38,8 @@
 time.
 
 
+.. _ref_s3transfer_usage:
+
 Usage
 =====
 
@@ -606,6 +608,11 @@ def __init__(self, client, config=None, osutil=None):
 
     def upload_file(self, filename, bucket, key,
                     callback=None, extra_args=None):
+        """Upload a file to an S3 object.
+
+        Variants have also been injected into S3 client, Bucket and Object.
+        You don't have to use S3Transfer.upload_file() directly.
+        """
         if extra_args is None:
             extra_args = {}
         self._validate_all_known_args(extra_args, self.ALLOWED_UPLOAD_ARGS)
@@ -636,11 +643,12 @@ def download_file(self, bucket, key, filename, extra_args=None,
                       callback=None):
         """Download an S3 object to a file.
 
-        This method will issue a ``head_object`` request to determine
-        the size of the S3 object.  This is used to determine if the
-        object is downloaded in parallel.
-
+        Variants have also been injected into S3 client, Bucket and Object.
+        You don't have to use S3Transfer.download_file() directly.
         """
+        # This method will issue a ``head_object`` request to determine
+        # the size of the S3 object.  This is used to determine if the
+        # object is downloaded in parallel.
         if extra_args is None:
             extra_args = {}
         self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS)

boto3/session.py

@@ -291,7 +291,11 @@ def _register_default_handlers(self):
         self._session.register(
             'creating-resource-class.s3.Bucket',
             boto3.utils.lazy_call(
-                'boto3.s3.inject.inject_bucket_load'))
+                'boto3.s3.inject.inject_bucket_methods'))
+        self._session.register(
+            'creating-resource-class.s3.Object',
+            boto3.utils.lazy_call(
+                'boto3.s3.inject.inject_object_methods'))
 
         # DynamoDb customizations
         self._session.register(

tests/functional/test_s3.py

@@ -29,3 +29,17 @@ def test_bucket_resource_has_load_method(self):
         bucket = session.resource('s3').Bucket('fakebucket')
         self.assertTrue(hasattr(bucket, 'load'),
                         'load() was not injected onto S3 Bucket resource.')
+
+    def test_transfer_methods_injected_to_bucket(self):
+        bucket = boto3.resource('s3').Bucket('my_bucket')
+        self.assertTrue(hasattr(bucket, 'upload_file'),
+                        'upload_file was not injected onto S3 bucket')
+        self.assertTrue(hasattr(bucket, 'download_file'),
+                        'download_file was not injected onto S3 bucket')
+
+    def test_transfer_methods_injected_to_object(self):
+        obj = boto3.resource('s3').Object('my_bucket', 'my_key')
+        self.assertTrue(hasattr(obj, 'upload_file'),
+                        'upload_file was not injected onto S3 object')
+        self.assertTrue(hasattr(obj, 'download_file'),
+                        'download_file was not injected onto S3 object')

tests/integration/test_s3.py

@@ -483,6 +483,28 @@ def test_transfer_methods_through_client(self):
                                   Filename=download_path)
         assert_files_equal(filename, download_path)
 
+    def test_transfer_methods_through_bucket(self):
+        # This is just a sanity check to ensure that the bucket interface work.
+        key = 'bucket.txt'
+        bucket = self.session.resource('s3').Bucket(self.bucket_name)
+        filename = self.files.create_file_with_size(key, 1024*1024)
+        bucket.upload_file(Filename=filename, Key=key)
+        self.addCleanup(self.delete_object, key)
+        download_path = os.path.join(self.files.rootdir, unique_id('foo'))
+        bucket.download_file(Key=key, Filename=download_path)
+        assert_files_equal(filename, download_path)
+
+    def test_transfer_methods_through_object(self):
+        # This is just a sanity check to ensure that the object interface work.
+        key = 'object.txt'
+        obj = self.session.resource('s3').Object(self.bucket_name, key)
+        filename = self.files.create_file_with_size(key, 1024*1024)
+        obj.upload_file(Filename=filename)
+        self.addCleanup(self.delete_object, key)
+        download_path = os.path.join(self.files.rootdir, unique_id('foo'))
+        obj.download_file(Filename=download_path)
+        assert_files_equal(filename, download_path)
+
 
 class TestCustomS3BucketLoad(unittest.TestCase):
     def setUp(self):

tests/unit/s3/test_inject.py

@@ -75,3 +75,39 @@ def test_bucket_load_raise_error(self):
         }
         with self.assertRaises(ClientError):
             inject.bucket_load(self.resource)
+
+
+class TestBucketTransferMethods(unittest.TestCase):
+
+    def setUp(self):
+        self.bucket = mock.Mock(name='my_bucket')
+
+    def test_upload_file_proxies_to_meta_client(self):
+        inject.bucket_upload_file(self.bucket, Filename='foo', Key='key')
+        self.bucket.meta.client.upload_file.assert_called_with(
+            Filename='foo', Bucket=self.bucket.name, Key='key',
+            ExtraArgs=None, Callback=None, Config=None)
+
+    def test_download_file_proxies_to_meta_client(self):
+        inject.bucket_download_file(self.bucket, Key='key', Filename='foo')
+        self.bucket.meta.client.download_file.assert_called_with(
+            Bucket=self.bucket.name, Key='key', Filename='foo',
+            ExtraArgs=None, Callback=None, Config=None)
+
+
+class TestObjectTransferMethods(unittest.TestCase):
+
+    def setUp(self):
+        self.obj = mock.Mock(bucket_name='my_bucket', key='my_key')
+
+    def test_upload_file_proxies_to_meta_client(self):
+        inject.object_upload_file(self.obj, Filename='foo')
+        self.obj.meta.client.upload_file.assert_called_with(
+            Filename='foo', Bucket=self.obj.bucket_name, Key=self.obj.key,
+            ExtraArgs=None, Callback=None, Config=None)
+
+    def test_download_file_proxies_to_meta_client(self):
+        inject.object_download_file(self.obj, Filename='foo')
+        self.obj.meta.client.download_file.assert_called_with(
+            Bucket=self.obj.bucket_name, Key=self.obj.key, Filename='foo',
+            ExtraArgs=None, Callback=None, Config=None)