Add generation arguments to bucket / blob methods. (#7444)

storage/google/cloud/storage/_helpers.py

@@ -43,8 +43,9 @@ class _PropertyMixin(object):
     """Abstract mixin for cloud storage classes with associated properties.
 
     Non-abstract subclasses should implement:
-      - client
       - path
+      - client
+      - user_project
 
     :type name: str
     :param name: The name of the object. Bucket names must start and end with a
@@ -98,6 +99,14 @@ def _encryption_headers(self):
         """
         return {}
 
+    @property
+    def _query_params(self):
+        """Default query parameters."""
+        params = {}
+        if self.user_project is not None:
+            params["userProject"] = self.user_project
+        return params
+
     def reload(self, client=None):
         """Reload properties from Cloud Storage.
 
@@ -109,17 +118,16 @@ def reload(self, client=None):
                        ``client`` stored on the current object.
         """
         client = self._require_client(client)
+        query_params = self._query_params
         # Pass only '?projection=noAcl' here because 'acl' and related
         # are handled via custom endpoints.
-        query_params = {"projection": "noAcl"}
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        query_params["projection"] = "noAcl"
         api_response = client._connection.api_request(
             method="GET",
             path=self.path,
             query_params=query_params,
             headers=self._encryption_headers(),
-            _target_object=self
+            _target_object=self,
         )
         self._set_properties(api_response)
 
@@ -164,11 +172,10 @@ def patch(self, client=None):
                        ``client`` stored on the current object.
         """
         client = self._require_client(client)
+        query_params = self._query_params
         # Pass '?projection=full' here because 'PATCH' documented not
         # to work properly w/ 'noAcl'.
-        query_params = {"projection": "full"}
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        query_params["projection"] = "full"
         update_properties = {key: self._properties[key] for key in self._changes}
 
         # Make the API call.
@@ -194,9 +201,8 @@ def update(self, client=None):
                        ``client`` stored on the current object.
         """
         client = self._require_client(client)
-        query_params = {"projection": "full"}
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        query_params = self._query_params
+        query_params["projection"] = "full"
         api_response = client._connection.api_request(
             method="PUT",
             path=self.path,

storage/google/cloud/storage/blob.py

@@ -159,7 +159,13 @@ class Blob(_PropertyMixin):
     """
 
     def __init__(
-        self, name, bucket, chunk_size=None, encryption_key=None, kms_key_name=None
+        self,
+        name,
+        bucket,
+        chunk_size=None,
+        encryption_key=None,
+        kms_key_name=None,
+        generation=None,
     ):
         name = _bytes_to_unicode(name)
         super(Blob, self).__init__(name=name)
@@ -177,6 +183,9 @@ def __init__(
         if kms_key_name is not None:
             self._properties["kmsKeyName"] = kms_key_name
 
+        if generation is not None:
+            self._properties["generation"] = generation
+
     @property
     def chunk_size(self):
         """Get the blob's default chunk size.
@@ -265,6 +274,16 @@ def _encryption_headers(self):
         """
         return _get_encryption_headers(self._encryption_key)
 
+    @property
+    def _query_params(self):
+        """Default query parameters."""
+        params = {}
+        if self.generation is not None:
+            params["generation"] = self.generation
+        if self.user_project is not None:
+            params["userProject"] = self.user_project
+        return params
+
     @property
     def public_url(self):
         """The public URL for this blob.
@@ -397,10 +416,8 @@ def exists(self, client=None):
         client = self._require_client(client)
         # We only need the status code (200 or not) so we seek to
         # minimize the returned payload.
-        query_params = {"fields": "name"}
-
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        query_params = self._query_params
+        query_params["fields"] = "name"
 
         try:
             # We intentionally pass `_target_object=None` since fields=name
@@ -435,7 +452,9 @@ def delete(self, client=None):
                  (propagated from
                  :meth:`google.cloud.storage.bucket.Bucket.delete_blob`).
         """
-        return self.bucket.delete_blob(self.name, client=client)
+        return self.bucket.delete_blob(
+            self.name, client=client, generation=self.generation
+        )
 
     def _get_transport(self, client):
         """Return the client's transport.
@@ -1492,13 +1511,15 @@ def rewrite(self, source, token=None, client=None):
         headers = _get_encryption_headers(self._encryption_key)
         headers.update(_get_encryption_headers(source._encryption_key, source=True))
 
-        query_params = {}
+        query_params = self._query_params
+        if "generation" in query_params:
+            del query_params["generation"]
 
         if token:
             query_params["rewriteToken"] = token
 
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        if source.generation:
+            query_params["sourceGeneration"] = source.generation
 
         if self.kms_key_name is not None:
             query_params["destinationKmsKeyName"] = self.kms_key_name

storage/google/cloud/storage/bucket.py

@@ -434,7 +434,14 @@ def user_project(self):
         """
         return self._user_project
 
-    def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=None):
+    def blob(
+        self,
+        blob_name,
+        chunk_size=None,
+        encryption_key=None,
+        kms_key_name=None,
+        generation=None,
+    ):
         """Factory constructor for blob object.
 
         .. note::
@@ -457,6 +464,10 @@ def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=Non
         :param kms_key_name:
             Optional resource name of KMS key used to encrypt blob's content.
 
+        :type generation: long
+        :param generation: Optional. If present, selects a specific revision of
+                           this object.
+
         :rtype: :class:`google.cloud.storage.blob.Blob`
         :returns: The blob object created.
         """
@@ -466,6 +477,7 @@ def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=Non
             chunk_size=chunk_size,
             encryption_key=encryption_key,
             kms_key_name=kms_key_name,
+            generation=generation,
         )
 
     def notification(
@@ -638,7 +650,9 @@ def path(self):
 
         return self.path_helper(self.name)
 
-    def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
+    def get_blob(
+        self, blob_name, client=None, encryption_key=None, generation=None, **kwargs
+    ):
         """Get a blob object by name.
 
         This will return None if the blob doesn't exist:
@@ -663,6 +677,10 @@ def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
             See
             https://cloud.google.com/storage/docs/encryption#customer-supplied.
 
+        :type generation: long
+        :param generation: Optional. If present, selects a specific revision of
+                           this object.
+
         :type kwargs: dict
         :param kwargs: Keyword arguments to pass to the
                        :class:`~google.cloud.storage.blob.Blob` constructor.
@@ -671,7 +689,11 @@ def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
         :returns: The blob object if it exists, otherwise None.
         """
         blob = Blob(
-            bucket=self, name=blob_name, encryption_key=encryption_key, **kwargs
+            bucket=self,
+            name=blob_name,
+            encryption_key=encryption_key,
+            generation=generation,
+            **kwargs
         )
         try:
             # NOTE: This will not fail immediately in a batch. However, when
@@ -867,7 +889,7 @@ def delete(self, force=False, client=None):
             _target_object=None,
         )
 
-    def delete_blob(self, blob_name, client=None):
+    def delete_blob(self, blob_name, client=None, generation=None):
         """Deletes a blob from the current bucket.
 
         If the blob isn't found (backend 404), raises a
@@ -889,6 +911,10 @@ def delete_blob(self, blob_name, client=None):
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the current bucket.
 
+        :type generation: long
+        :param generation: Optional. If present, permanently deletes a specific
+                           revision of this object.
+
         :raises: :class:`google.cloud.exceptions.NotFound` (to suppress
                  the exception, call ``delete_blobs``, passing a no-op
                  ``on_error`` callback, e.g.:
@@ -899,19 +925,15 @@ def delete_blob(self, blob_name, client=None):
 
         """
         client = self._require_client(client)
-        query_params = {}
+        blob = Blob(blob_name, bucket=self, generation=generation)
 
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
-
-        blob_path = Blob.path_helper(self.path, blob_name)
         # We intentionally pass `_target_object=None` since a DELETE
         # request has no response value (whether in a standard request or
         # in a batch request).
         client._connection.api_request(
             method="DELETE",
-            path=blob_path,
-            query_params=query_params,
+            path=blob.path,
+            query_params=blob._query_params,
             _target_object=None,
         )
 

storage/tests/system.py

@@ -44,7 +44,9 @@ def _bad_copy(bad_request):
 
 
 retry_429 = RetryErrors(exceptions.TooManyRequests, max_tries=6)
-retry_429_503 = RetryErrors([exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=6)
+retry_429_503 = RetryErrors(
+    [exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=6
+)
 retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)
 
 
@@ -78,6 +80,7 @@ def setUpModule():
     # In the **very** rare case the bucket name is reserved, this
     # fails with a ConnectionError.
     Config.TEST_BUCKET = Config.CLIENT.bucket(bucket_name)
+    Config.TEST_BUCKET.versioning_enabled = True
     retry_429(Config.TEST_BUCKET.create)()
 
 
@@ -414,29 +417,52 @@ def test_crud_blob_w_user_project(self):
 
         # Exercise 'objects.insert' w/ userProject.
         blob.upload_from_filename(file_data["path"])
+        gen0 = blob.generation
+
+        # Upload a second generation of the blob
+        blob.upload_from_string(b"gen1")
+        gen1 = blob.generation
+
+        blob0 = with_user_project.blob("SmallFile", generation=gen0)
+        blob1 = with_user_project.blob("SmallFile", generation=gen1)
+
+        # Exercise 'objects.get' w/ generation
+        self.assertEqual(with_user_project.get_blob(blob.name).generation, gen1)
+        self.assertEqual(
+            with_user_project.get_blob(blob.name, generation=gen0).generation, gen0
+        )
 
         try:
             # Exercise 'objects.get' (metadata) w/ userProject.
             self.assertTrue(blob.exists())
             blob.reload()
 
             # Exercise 'objects.get' (media) w/ userProject.
-            downloaded = blob.download_as_string()
-            self.assertEqual(downloaded, file_contents)
+            self.assertEqual(blob0.download_as_string(), file_contents)
+            self.assertEqual(blob1.download_as_string(), b"gen1")
 
             # Exercise 'objects.patch' w/ userProject.
-            blob.content_language = "en"
-            blob.patch()
-            self.assertEqual(blob.content_language, "en")
+            blob0.content_language = "en"
+            blob0.patch()
+            self.assertEqual(blob0.content_language, "en")
+            self.assertIsNone(blob1.content_language)
 
             # Exercise 'objects.update' w/ userProject.
             metadata = {"foo": "Foo", "bar": "Bar"}
-            blob.metadata = metadata
-            blob.update()
-            self.assertEqual(blob.metadata, metadata)
+            blob0.metadata = metadata
+            blob0.update()
+            self.assertEqual(blob0.metadata, metadata)
+            self.assertIsNone(blob1.metadata)
         finally:
             # Exercise 'objects.delete' (metadata) w/ userProject.
-            blob.delete()
+            blobs = with_user_project.list_blobs(prefix=blob.name, versions=True)
+            self.assertEqual([each.generation for each in blobs], [gen0, gen1])
+
+            blob0.delete()
+            blobs = with_user_project.list_blobs(prefix=blob.name, versions=True)
+            self.assertEqual([each.generation for each in blobs], [gen1])
+
+            blob1.delete()
 
     @unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.")
     def test_blob_acl_w_user_project(self):

storage/tests/unit/test__helpers.py

@@ -59,6 +59,15 @@ def test__encryption_headers(self):
         mixin = self._make_one()
         self.assertEqual(mixin._encryption_headers(), {})
 
+    def test__query_params_wo_user_project(self):
+        derived = self._derivedClass("/path", None)()
+        self.assertEqual(derived._query_params, {})
+
+    def test__query_params_w_user_project(self):
+        user_project = "user-project-123"
+        derived = self._derivedClass("/path", user_project)()
+        self.assertEqual(derived._query_params, {"userProject": user_project})
+
     def test_reload(self):
         connection = _Connection({"foo": "Foo"})
         client = _Client(connection)