Address #233: support blob_value in protobuf

gcloud/datastore/_helpers.py

@@ -29,6 +29,11 @@ def _get_protobuf_attribute_and_value(val):
     `gcloud.datastore.key.Key` into a Protobuf representation.
     This function handles that for you.
 
+    .. note::
+       Values which are "text" ('unicode' in Python2, 'str' in Python3) map
+       to 'string_value' in the datastore;  values which are "bytes"
+       ('str' in Python2, 'bytes' in Python3) map to 'blob_value'.
+
     For example:
 
     >>> _get_protobuf_attribute_and_value(1234)
@@ -62,8 +67,10 @@ def _get_protobuf_attribute_and_value(val):
     elif isinstance(val, (int, long)):
         INT_VALUE_CHECKER.CheckValue(val)   # Raise an exception if invalid.
         name, value = 'integer', long(val)  # Always cast to a long.
-    elif isinstance(val, basestring):
+    elif isinstance(val, unicode):
         name, value = 'string', val
+    elif isinstance(val, (bytes, str)):
+        name, value = 'blob', val
     elif isinstance(val, Entity):
         name, value = 'entity', val
     elif isinstance(val, list):
@@ -112,6 +119,9 @@ def _get_value_from_value_pb(value_pb):
     elif value_pb.HasField('string_value'):
         result = value_pb.string_value
 
+    elif value_pb.HasField('blob_value'):
+        result = value_pb.blob_value
+
     elif value_pb.HasField('entity_value'):
         result = Entity.from_protobuf(value_pb.entity_value)
 

gcloud/datastore/entity.py

@@ -208,6 +208,11 @@ def save(self):
            not correspond to keys set on this instance will be removed from
            the datastore.
 
+        .. note::
+           Property values which are "text" ('unicode' in Python2, 'str' in
+           Python3) map to 'string_value' in the datastore;  values which are
+           "bytes" ('str' in Python2, 'bytes' in Python3) map to 'blob_value'.
+
         :rtype: :class:`gcloud.datastore.entity.Entity`
         :returns: The entity with a possibly updated Key.
         """

gcloud/datastore/test__helpers.py

@@ -75,9 +75,14 @@ def test_long_too_large(self):
 
     def test_native_str(self):
         name, value = self._callFUT('str')
-        self.assertEqual(name, 'string_value')
+        self.assertEqual(name, 'blob_value')
         self.assertEqual(value, 'str')
 
+    def test_bytes(self):
+        name, value = self._callFUT(b'bytes')
+        self.assertEqual(name, 'blob_value')
+        self.assertEqual(value, b'bytes')
+
     def test_unicode(self):
         name, value = self._callFUT(u'str')
         self.assertEqual(name, 'string_value')
@@ -151,9 +156,9 @@ def test_int(self):
         pb = self._makePB('integer_value', 42)
         self.assertEqual(self._callFUT(pb), 42)
 
-    def test_native_str(self):
-        pb = self._makePB('string_value', 'str')
-        self.assertEqual(self._callFUT(pb), 'str')
+    def test_bytes(self):
+        pb = self._makePB('blob_value', b'str')
+        self.assertEqual(self._callFUT(pb), b'str')
 
     def test_unicode(self):
         pb = self._makePB('string_value', u'str')
@@ -272,9 +277,15 @@ def test_long(self):
     def test_native_str(self):
         pb = self._makePB()
         self._callFUT(pb, 'str')
-        value = pb.string_value
+        value = pb.blob_value
         self.assertEqual(value, 'str')
 
+    def test_bytes(self):
+        pb = self._makePB()
+        self._callFUT(pb, b'str')
+        value = pb.blob_value
+        self.assertEqual(value, b'str')
+
     def test_unicode(self):
         pb = self._makePB()
         self._callFUT(pb, u'str')
@@ -299,18 +310,18 @@ def test_entity_w_key(self):
         pb = self._makePB()
         key = Key(path=[{'kind': 'KIND', 'id': 123}])
         entity = Entity().key(key)
-        entity['foo'] = 'Foo'
+        entity['foo'] = u'Foo'
         self._callFUT(pb, entity)
         value = pb.entity_value
         self.assertEqual(value.key, key.to_protobuf())
         props = list(value.property)
         self.assertEqual(len(props), 1)
         self.assertEqual(props[0].name, 'foo')
-        self.assertEqual(props[0].value.string_value, 'Foo')
+        self.assertEqual(props[0].value.string_value, u'Foo')
 
     def test_list(self):
         pb = self._makePB()
-        values = ['a', 0, 3.14]
+        values = [u'a', 0, 3.14]
         self._callFUT(pb, values)
         marshalled = pb.list_value
         self.assertEqual(len(marshalled), len(values))

gcloud/datastore/test_connection.py

@@ -530,7 +530,7 @@ def test_commit_wo_transaction(self):
         insert.key.CopyFrom(key_pb)
         prop = insert.property.add()
         prop.name = 'foo'
-        prop.value.string_value = 'Foo'
+        prop.value.string_value = u'Foo'
         conn = self._makeOne()
         URI = '/'.join([
             conn.API_BASE_URL,
@@ -577,7 +577,7 @@ def id(self):
         insert.key.CopyFrom(key_pb)
         prop = insert.property.add()
         prop.name = 'foo'
-        prop.value.string_value = 'Foo'
+        prop.value.string_value = u'Foo'
         conn = self._makeOne()
         conn.transaction(Xact())
         URI = '/'.join([
@@ -627,7 +627,7 @@ def test_save_entity_wo_transaction_w_upsert(self):
             'commit',
         ])
         http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString())
-        result = conn.save_entity(DATASET_ID, key_pb, {'foo': 'Foo'})
+        result = conn.save_entity(DATASET_ID, key_pb, {'foo': u'Foo'})
         self.assertEqual(result, True)
         cw = http._called_with
         self.assertEqual(cw['uri'], URI)
@@ -651,7 +651,7 @@ def test_save_entity_wo_transaction_w_upsert(self):
         props = list(upsert.property)
         self.assertEqual(len(props), 1)
         self.assertEqual(props[0].name, 'foo')
-        self.assertEqual(props[0].value.string_value, 'Foo')
+        self.assertEqual(props[0].value.string_value, u'Foo')
         self.assertEqual(len(mutation.delete), 0)
         self.assertEqual(request.mode, rq_class.NON_TRANSACTIONAL)
 
@@ -680,7 +680,7 @@ def test_save_entity_wo_transaction_w_auto_id(self):
             'commit',
         ])
         http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString())
-        result = conn.save_entity(DATASET_ID, key_pb, {'foo': 'Foo'})
+        result = conn.save_entity(DATASET_ID, key_pb, {'foo': u'Foo'})
         self.assertEqual(result, updated_key_pb)
         cw = http._called_with
         self.assertEqual(cw['uri'], URI)
@@ -702,7 +702,7 @@ def test_save_entity_wo_transaction_w_auto_id(self):
         props = list(insert.property)
         self.assertEqual(len(props), 1)
         self.assertEqual(props[0].name, 'foo')
-        self.assertEqual(props[0].value.string_value, 'Foo')
+        self.assertEqual(props[0].value.string_value, u'Foo')
         self.assertEqual(len(inserts), 1)
         upserts = list(mutation.upsert)
         self.assertEqual(len(upserts), 0)
@@ -726,7 +726,7 @@ def mutation(self):
         conn = self._makeOne()
         conn.transaction(Xact())
         http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString())
-        result = conn.save_entity(DATASET_ID, key_pb, {'foo': 'Foo'})
+        result = conn.save_entity(DATASET_ID, key_pb, {'foo': u'Foo'})
         self.assertEqual(result, True)
         self.assertEqual(http._called_with, None)
         mutation = conn.mutation()
@@ -745,7 +745,7 @@ def mutation(self):
                 return mutation
         DATASET_ID = 'DATASET'
         nested = Entity()
-        nested['bar'] = 'Bar'
+        nested['bar'] = u'Bar'
         key_pb = Key(dataset=Dataset(DATASET_ID),
                      path=[{'kind': 'Kind', 'id': 1234}]).to_protobuf()
         rsp_pb = datastore_pb.CommitResponse()

gcloud/datastore/test_query.py

@@ -64,23 +64,23 @@ def test_filter_w_unknown_operator(self):
 
     def test_filter_w_known_operator(self):
         query = self._makeOne()
-        after = query.filter('firstname =', 'John')
+        after = query.filter('firstname =', u'John')
         self.assertFalse(after is query)
         self.assertTrue(isinstance(after, self._getTargetClass()))
         q_pb = after.to_protobuf()
         self.assertEqual(q_pb.filter.composite_filter.operator, 1)  # AND
         f_pb, = list(q_pb.filter.composite_filter.filter)
         p_pb = f_pb.property_filter
         self.assertEqual(p_pb.property.name, 'firstname')
-        self.assertEqual(p_pb.value.string_value, 'John')
+        self.assertEqual(p_pb.value.string_value, u'John')
 
     def test_filter_w_known_operator_and_entity(self):
         import operator
         from gcloud.datastore.entity import Entity
         query = self._makeOne()
         other = Entity()
-        other['firstname'] = 'John'
-        other['lastname'] = 'Smith'
+        other['firstname'] = u'John'
+        other['lastname'] = u'Smith'
         after = query.filter('other =', other)
         self.assertFalse(after is query)
         self.assertTrue(isinstance(after, self._getTargetClass()))
@@ -93,9 +93,9 @@ def test_filter_w_known_operator_and_entity(self):
         props = sorted(other_pb.property, key=operator.attrgetter('name'))
         self.assertEqual(len(props), 2)
         self.assertEqual(props[0].name, 'firstname')
-        self.assertEqual(props[0].value.string_value, 'John')
+        self.assertEqual(props[0].value.string_value, u'John')
         self.assertEqual(props[1].name, 'lastname')
-        self.assertEqual(props[1].value.string_value, 'Smith')
+        self.assertEqual(props[1].value.string_value, u'Smith')
 
     def test_ancestor_w_non_key_non_list(self):
         query = self._makeOne()
@@ -105,7 +105,7 @@ def test_ancester_wo_existing_ancestor_query_w_key_and_propfilter(self):
         from gcloud.datastore.key import Key
         _KIND = 'KIND'
         _ID = 123
-        _NAME = 'NAME'
+        _NAME = u'NAME'
         key = Key(path=[{'kind': _KIND, 'id': _ID}])
         query = self._makeOne().filter('name =', _NAME)
         after = query.ancestor(key)
@@ -167,7 +167,7 @@ def test_ancester_clears_existing_ancestor_query_w_only(self):
     def test_ancester_clears_existing_ancestor_query_w_others(self):
         _KIND = 'KIND'
         _ID = 123
-        _NAME = 'NAME'
+        _NAME = u'NAME'
         query = self._makeOne().filter('name =', _NAME)
         between = query.ancestor([_KIND, _ID])
         after = between.ancestor(None)
@@ -246,7 +246,7 @@ def test_fetch_default_limit(self):
         path_element.id = _ID
         prop = entity_pb.property.add()
         prop.name = 'foo'
-        prop.value.string_value = 'Foo'
+        prop.value.string_value = u'Foo'
         connection = _Connection(entity_pb)
         dataset = _Dataset(_DATASET, connection)
         query = self._makeOne(_KIND, dataset)
@@ -272,7 +272,7 @@ def test_fetch_explicit_limit(self):
         path_element.id = _ID
         prop = entity_pb.property.add()
         prop.name = 'foo'
-        prop.value.string_value = 'Foo'
+        prop.value.string_value = u'Foo'
         connection = _Connection(entity_pb)
         connection._cursor = _CURSOR
         dataset = _Dataset(_DATASET, connection)