From 72d85f227f405e33961ae7b53f8d8c7cbe86a523 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 3 Nov 2014 15:58:02 -0500 Subject: [PATCH 1/3] Add 'exclude_from_indexes' method to Entity. Set it via ctor argument. Pass it to 'Connection.save_entity'. Fields in the sequence will have the 'indexed' field set False in the corrsponding protobuf. Fixes #83. --- gcloud/datastore/connection.py | 9 +++++- gcloud/datastore/entity.py | 13 +++++++-- gcloud/datastore/test_connection.py | 45 +++++++++++++++++++++++++++++ gcloud/datastore/test_entity.py | 28 +++++++++++------- 4 files changed, 81 insertions(+), 14 deletions(-) diff --git a/gcloud/datastore/connection.py b/gcloud/datastore/connection.py index 5bcdcccd5ca1..2840c787616f 100644 --- a/gcloud/datastore/connection.py +++ b/gcloud/datastore/connection.py @@ -371,7 +371,8 @@ def allocate_ids(self, dataset_id, key_pbs): datastore_pb.AllocateIdsResponse) return list(response.key) - def save_entity(self, dataset_id, key_pb, properties): + def save_entity(self, dataset_id, key_pb, properties, + exclude_from_indexes=()): """Save an entity to the Cloud Datastore with the provided properties. .. note:: @@ -387,6 +388,9 @@ def save_entity(self, dataset_id, key_pb, properties): :type properties: dict :param properties: The properties to store on the entity. + + :type exclude_from_indexes: sequence of str + :param exclude_from_indexes: Names of properties *not* to be indexed. """ mutation = self.mutation() @@ -410,6 +414,9 @@ def save_entity(self, dataset_id, key_pb, properties): # Set the appropriate value. helpers._set_protobuf_value(prop.value, value) + if name in exclude_from_indexes: + prop.value.indexed = False + # If this is in a transaction, we should just return True. The # transaction will handle assigning any keys as necessary. if self.transaction(): diff --git a/gcloud/datastore/entity.py b/gcloud/datastore/entity.py index cf325d8b0fba..295792e28bbd 100644 --- a/gcloud/datastore/entity.py +++ b/gcloud/datastore/entity.py @@ -80,13 +80,14 @@ class Entity(dict): """ - def __init__(self, dataset=None, kind=None): + def __init__(self, dataset=None, kind=None, exclude_from_indexes=()): super(Entity, self).__init__() self._dataset = dataset if kind: self._key = Key().kind(kind) else: self._key = None + self._exclude_from_indexes = set(exclude_from_indexes) def dataset(self): """Get the :class:`.dataset.Dataset` in which this entity belongs. @@ -140,6 +141,13 @@ def kind(self): if self._key: return self._key.kind() + def exclude_from_indexes(self): + """Return field names which are *not* to be indexed. + + :rtype: list(str) + """ + return frozenset(self._exclude_from_indexes) + @classmethod def from_key(cls, key, dataset=None): """Create entity based on :class:`.datastore.key.Key`. @@ -223,7 +231,8 @@ def save(self): key_pb = connection.save_entity( dataset_id=dataset.id(), key_pb=key.to_protobuf(), - properties=dict(self)) + properties=dict(self), + exclude_from_indexes=self.exclude_from_indexes()) # If we are in a transaction and the current entity needs an # automatically assigned ID, tell the transaction where to put that. diff --git a/gcloud/datastore/test_connection.py b/gcloud/datastore/test_connection.py index 53db7d65a8a9..d425d4654184 100644 --- a/gcloud/datastore/test_connection.py +++ b/gcloud/datastore/test_connection.py @@ -665,6 +665,51 @@ def test_save_entity_wo_transaction_w_upsert(self): self.assertEqual(len(props), 1) self.assertEqual(props[0].name, 'foo') self.assertEqual(props[0].value.string_value, u'Foo') + self.assertEqual(props[0].value.indexed, True) + self.assertEqual(len(mutation.delete), 0) + self.assertEqual(request.mode, rq_class.NON_TRANSACTIONAL) + + def test_save_entity_w_exclude_from_indexes(self): + from gcloud.datastore.connection import datastore_pb + from gcloud.datastore.key import Key + + DATASET_ID = 'DATASET' + key_pb = Key(path=[{'kind': 'Kind', 'id': 1234}]).to_protobuf() + rsp_pb = datastore_pb.CommitResponse() + conn = self._makeOne() + URI = '/'.join([ + conn.API_BASE_URL, + 'datastore', + conn.API_VERSION, + 'datasets', + DATASET_ID, + 'commit', + ]) + http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString()) + result = conn.save_entity(DATASET_ID, key_pb, {'foo': u'Foo'}, + exclude_from_indexes=['foo']) + self.assertEqual(result, True) + cw = http._called_with + self.assertEqual(cw['uri'], URI) + self.assertEqual(cw['method'], 'POST') + self.assertEqual(cw['headers']['Content-Type'], + 'application/x-protobuf') + self.assertEqual(cw['headers']['User-Agent'], conn.USER_AGENT) + rq_class = datastore_pb.CommitRequest + request = rq_class() + request.ParseFromString(cw['body']) + self.assertEqual(request.transaction, '') + mutation = request.mutation + self.assertEqual(len(mutation.insert_auto_id), 0) + upserts = list(mutation.upsert) + self.assertEqual(len(upserts), 1) + upsert = upserts[0] + self.assertEqual(upsert.key, key_pb) + props = list(upsert.property) + self.assertEqual(len(props), 1) + self.assertEqual(props[0].name, 'foo') + self.assertEqual(props[0].value.string_value, u'Foo') + self.assertEqual(props[0].value.indexed, False) self.assertEqual(len(mutation.delete), 0) self.assertEqual(request.mode, rq_class.NON_TRANSACTIONAL) diff --git a/gcloud/datastore/test_entity.py b/gcloud/datastore/test_entity.py index d53f32e6e7d0..6f796622593f 100644 --- a/gcloud/datastore/test_entity.py +++ b/gcloud/datastore/test_entity.py @@ -13,13 +13,13 @@ def _getTargetClass(self): return Entity - def _makeOne(self, dataset=_MARKER, kind=_KIND): + def _makeOne(self, dataset=_MARKER, kind=_KIND, exclude_from_indexes=()): from gcloud.datastore.dataset import Dataset klass = self._getTargetClass() if dataset is _MARKER: dataset = Dataset(_DATASET_ID) - return klass(dataset, kind) + return klass(dataset, kind, exclude_from_indexes) def test_ctor_defaults(self): klass = self._getTargetClass() @@ -27,13 +27,17 @@ def test_ctor_defaults(self): self.assertEqual(entity.key(), None) self.assertEqual(entity.dataset(), None) self.assertEqual(entity.kind(), None) + self.assertEqual(sorted(entity.exclude_from_indexes()), []) def test_ctor_explicit(self): from gcloud.datastore.dataset import Dataset dataset = Dataset(_DATASET_ID) - entity = self._makeOne(dataset, _KIND) + _EXCLUDE_FROM_INDEXES = ['foo', 'bar'] + entity = self._makeOne(dataset, _KIND, _EXCLUDE_FROM_INDEXES) self.assertTrue(entity.dataset() is dataset) + self.assertEqual(sorted(entity.exclude_from_indexes()), + sorted(_EXCLUDE_FROM_INDEXES)) def test_key_getter(self): from gcloud.datastore.key import Key @@ -132,7 +136,7 @@ def test_save_wo_transaction_wo_auto_id_wo_returned_key(self): self.assertTrue(entity.save() is entity) self.assertEqual(entity['foo'], 'Foo') self.assertEqual(connection._saved, - (_DATASET_ID, 'KEY', {'foo': 'Foo'})) + (_DATASET_ID, 'KEY', {'foo': 'Foo'}, ())) self.assertEqual(key._path, None) def test_save_w_transaction_wo_partial_key(self): @@ -146,7 +150,7 @@ def test_save_w_transaction_wo_partial_key(self): self.assertTrue(entity.save() is entity) self.assertEqual(entity['foo'], 'Foo') self.assertEqual(connection._saved, - (_DATASET_ID, 'KEY', {'foo': 'Foo'})) + (_DATASET_ID, 'KEY', {'foo': 'Foo'}, ())) self.assertEqual(transaction._added, ()) self.assertEqual(key._path, None) @@ -162,11 +166,11 @@ def test_save_w_transaction_w_partial_key(self): self.assertTrue(entity.save() is entity) self.assertEqual(entity['foo'], 'Foo') self.assertEqual(connection._saved, - (_DATASET_ID, 'KEY', {'foo': 'Foo'})) + (_DATASET_ID, 'KEY', {'foo': 'Foo'}, ())) self.assertEqual(transaction._added, (entity,)) self.assertEqual(key._path, None) - def test_save_w_returned_key(self): + def test_save_w_returned_key_exclude_from_indexes(self): from gcloud.datastore import datastore_v1_pb2 as datastore_pb key_pb = datastore_pb.Key() key_pb.partition_id.dataset_id = _DATASET_ID @@ -175,13 +179,13 @@ def test_save_w_returned_key(self): connection._save_result = key_pb dataset = _Dataset(connection) key = _Key() - entity = self._makeOne(dataset) + entity = self._makeOne(dataset, exclude_from_indexes=['foo']) entity.key(key) entity['foo'] = 'Foo' self.assertTrue(entity.save() is entity) self.assertEqual(entity['foo'], 'Foo') self.assertEqual(connection._saved, - (_DATASET_ID, 'KEY', {'foo': 'Foo'})) + (_DATASET_ID, 'KEY', {'foo': 'Foo'}, ('foo',))) self.assertEqual(key._path, [{'kind': _KIND, 'id': _ID}]) def test_delete_no_key(self): @@ -257,8 +261,10 @@ class _Connection(object): def transaction(self): return self._transaction - def save_entity(self, dataset_id, key_pb, properties): - self._saved = (dataset_id, key_pb, properties) + def save_entity(self, dataset_id, key_pb, properties, + exclude_from_indexes=()): + self._saved = (dataset_id, key_pb, properties, + tuple(exclude_from_indexes)) return self._save_result def delete_entities(self, dataset_id, key_pbs): From 41a041260e7b687a1426993655a3ad8dd15ccf09 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 3 Nov 2014 17:10:55 -0500 Subject: [PATCH 2/3] Expose 'exclude_from_indexes' via 'Dataset.entity()'. Incorporates feedback from @dhermes. --- gcloud/datastore/dataset.py | 9 +++++++-- gcloud/datastore/test_dataset.py | 13 ++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/gcloud/datastore/dataset.py b/gcloud/datastore/dataset.py index cb5484abbf38..f72aeb0ab971 100644 --- a/gcloud/datastore/dataset.py +++ b/gcloud/datastore/dataset.py @@ -78,16 +78,21 @@ def query(self, *args, **kwargs): kwargs['dataset'] = self return Query(*args, **kwargs) - def entity(self, kind): + def entity(self, kind, exclude_from_indexes=()): """Create an entity bound to this dataset. :type kind: string :param kind: the "kind" of the new entity. + :type exclude_from_indexes: sequence of str + :param exclude_from_indexes: Names of properties *not* to be indexed + for the entity. + :rtype: :class:`gcloud.datastore.entity.Entity` :returns: a new Entity instance, bound to this dataset. """ - return Entity(dataset=self, kind=kind) + return Entity(dataset=self, kind=kind, + exclude_from_indexes=exclude_from_indexes) def transaction(self, *args, **kwargs): """Create a transaction bound to this dataset. diff --git a/gcloud/datastore/test_dataset.py b/gcloud/datastore/test_dataset.py index 588499c2f41a..0dc70c3f1900 100644 --- a/gcloud/datastore/test_dataset.py +++ b/gcloud/datastore/test_dataset.py @@ -34,7 +34,7 @@ def test_query_factory(self): self.assertIsInstance(query, Query) self.assertTrue(query.dataset() is dataset) - def test_entity_factory(self): + def test_entity_factory_defaults(self): from gcloud.datastore.entity import Entity DATASET_ID = 'DATASET' KIND = 'KIND' @@ -42,6 +42,17 @@ def test_entity_factory(self): entity = dataset.entity(KIND) self.assertIsInstance(entity, Entity) self.assertEqual(entity.kind(), KIND) + self.assertEqual(sorted(entity.exclude_from_indexes()), []) + + def test_entity_factory_explicit(self): + from gcloud.datastore.entity import Entity + DATASET_ID = 'DATASET' + KIND = 'KIND' + dataset = self._makeOne(DATASET_ID) + entity = dataset.entity(KIND, ['foo', 'bar']) + self.assertIsInstance(entity, Entity) + self.assertEqual(entity.kind(), KIND) + self.assertEqual(sorted(entity.exclude_from_indexes()), ['bar', 'foo']) def test_transaction_factory(self): from gcloud.datastore.transaction import Transaction From 5762cbfaae2e474943bf87272e7014fb7d431ff9 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 4 Nov 2014 12:35:09 -0500 Subject: [PATCH 3/3] Avoid mandating type of field names passed to 'exclude_from_indexes'. Incorporates feedback from @dhermes. --- gcloud/datastore/dataset.py | 8 ++++---- gcloud/datastore/entity.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/gcloud/datastore/dataset.py b/gcloud/datastore/dataset.py index f72aeb0ab971..674290e59e0a 100644 --- a/gcloud/datastore/dataset.py +++ b/gcloud/datastore/dataset.py @@ -82,11 +82,11 @@ def entity(self, kind, exclude_from_indexes=()): """Create an entity bound to this dataset. :type kind: string - :param kind: the "kind" of the new entity. + :param kind: the "kind" of the new entity (see + https://cloud.google.com/datastore/docs/concepts/entities#Datastore_Kinds_and_identifiers) - :type exclude_from_indexes: sequence of str - :param exclude_from_indexes: Names of properties *not* to be indexed - for the entity. + :param exclude_from_indexes: names of fields whose values are not to + be indexed. :rtype: :class:`gcloud.datastore.entity.Entity` :returns: a new Entity instance, bound to this dataset. diff --git a/gcloud/datastore/entity.py b/gcloud/datastore/entity.py index 295792e28bbd..96c4c887f937 100644 --- a/gcloud/datastore/entity.py +++ b/gcloud/datastore/entity.py @@ -78,6 +78,15 @@ class Entity(dict): Python3), will be saved using the 'blob_value' field, without any decoding / encoding step. + :type dataset: :class:`gcloud.datastore.dataset.Dataset`, or None + :param dataset: the Dataset instance associated with this entity. + + :type kind: str + :param kind: the "kind" of the entity (see + https://cloud.google.com/datastore/docs/concepts/entities#Datastore_Kinds_and_identifiers) + + :param exclude_from_indexes: names of fields whose values are not to be + indexed for this entity. """ def __init__(self, dataset=None, kind=None, exclude_from_indexes=()): @@ -142,9 +151,9 @@ def kind(self): return self._key.kind() def exclude_from_indexes(self): - """Return field names which are *not* to be indexed. + """Names of fields which are *not* to be indexed for this entity. - :rtype: list(str) + :rtype: sequence of field names """ return frozenset(self._exclude_from_indexes)