From 55702ca1620abf93d52124fb96025821ef8b15a3 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Mon, 22 Nov 2021 12:04:35 -0800 Subject: [PATCH 1/3] samples: add batch request sample and test --- samples/snippets/snippets_test.py | 15 ++++++ samples/snippets/storage_batch_request.py | 59 +++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 samples/snippets/storage_batch_request.py diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index dd8e6aeaf..ad20a00b8 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -23,6 +23,7 @@ import requests import storage_add_bucket_label +import storage_batch_request import storage_bucket_delete_default_kms_key import storage_change_default_storage_class import storage_change_file_storage_class @@ -509,3 +510,17 @@ def test_storage_configure_retries(test_blob, capsys): assert "The following library method is customized to be retried" in out assert "_should_retry" in out assert "initial=1.5, maximum=45.0, multiplier=1.2, deadline=500.0" in out + + +def test_batch_request(test_bucket): + blob1 = test_bucket.blob("b/1.txt") + blob2 = test_bucket.blob("b/2.txt") + blob1.upload_from_string("hello world") + blob2.upload_from_string("hello world") + + storage_batch_request.batch_request(test_bucket.name, "b/") + blob1.reload() + blob2.reload() + + assert blob1.metadata.get("your-metadata-key") == "your-metadata-value" + assert blob2.metadata.get("your-metadata-key") == "your-metadata-value" diff --git a/samples/snippets/storage_batch_request.py b/samples/snippets/storage_batch_request.py new file mode 100644 index 000000000..2e2d0ba90 --- /dev/null +++ b/samples/snippets/storage_batch_request.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that uses a batch request. +This sample is used on this page: + https://cloud.google.com/storage/docs/batch +For more information, see README.md. +""" + +# [START storage_batch_request] + +from google.cloud import storage + + +def batch_request(bucket_name, prefix=None): + """Use a batch request to patch a list of objects with the given prefix in a bucket.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + # The prefix of the object paths + # prefix = "directory-prefix/" + + client = storage.Client() + bucket = client.bucket(bucket_name) + + # Accumulate in a list the objects with a given prefix. + blobs_to_patch = [blob for blob in bucket.list_blobs(prefix=prefix)] + + # Use a batch context manager to edit metadata in the list of blobs. + # The batch request is sent out when the context manager closes. + with client.batch(): + for blob in blobs_to_patch: + metadata = {"your-metadata-key": "your-metadata-value"} + blob.metadata = metadata + blob.patch() + + print( + f"Batch request edited metadata for all objects with the given prefix in {bucket.name}." + ) + + +# [END storage_batch_request] + +if __name__ == "__main__": + batch_request(bucket_name=sys.argv[1], prefix=sys.argv[2]) From f2669958ba55f32cc1ead192b5a12fa746057c6c Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Mon, 22 Nov 2021 13:27:26 -0800 Subject: [PATCH 2/3] samples: update readme with new sample --- samples/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/samples/README.md b/samples/README.md index bb48adc9b..a0d3f9781 100644 --- a/samples/README.md +++ b/samples/README.md @@ -49,6 +49,7 @@ for instructions on setting up credentials for applications. List of Samples * [Activate HMAC Key](#activate-hmac-key) +* [Batch Request](#batch-request) * [Add Bucket Conditional IAM Binding](#add-bucket-conditional-iam-binding) * [Add Bucket Default Owner](#add-bucket-default-owner) * [Add Bucket IAM Member](#add-bucket-iam-member) @@ -158,6 +159,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_activate_hmac_key.py ` +----- +### Batch Request +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_batch_request.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_batch_request.py). To run this sample: + + +`python storage_batch_request.py ` + ----- ### Add Bucket Conditional IAM Binding From 17bc742b05fe93102949d05fd918e3e381a723d8 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Thu, 2 Dec 2021 13:45:22 -0800 Subject: [PATCH 3/3] add clarifying comment --- samples/snippets/storage_batch_request.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/snippets/storage_batch_request.py b/samples/snippets/storage_batch_request.py index 2e2d0ba90..863fc09cd 100644 --- a/samples/snippets/storage_batch_request.py +++ b/samples/snippets/storage_batch_request.py @@ -42,6 +42,7 @@ def batch_request(bucket_name, prefix=None): # Use a batch context manager to edit metadata in the list of blobs. # The batch request is sent out when the context manager closes. + # No more than 100 calls should be included in a single batch request. with client.batch(): for blob in blobs_to_patch: metadata = {"your-metadata-key": "your-metadata-value"}