From 217c2f259a0d694f5a03a4e26d8ff78416845ef3 Mon Sep 17 00:00:00 2001 From: Steffen Oschatz Date: Fri, 18 Nov 2016 13:41:15 +0100 Subject: [PATCH 1/5] added head_only parameter for pbc transport --- riak/bucket.py | 8 ++++++-- riak/client/operations.py | 8 ++++++-- riak/codecs/pbuf.py | 4 +++- riak/content.py | 2 ++ riak/riak_object.py | 7 +++++-- riak/tests/test_kv.py | 7 +++++++ riak/transports/tcp/transport.py | 4 ++-- riak/transports/transport.py | 2 +- 8 files changed, 32 insertions(+), 10 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index f6dd3863..be499441 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -194,7 +194,7 @@ def new(self, key=None, data=None, content_type='application/json', return obj def get(self, key, r=None, pr=None, timeout=None, include_context=None, - basic_quorum=None, notfound_ok=None): + basic_quorum=None, notfound_ok=None, head_only=False): """ Retrieve a :class:`~riak.riak_object.RiakObject` or :class:`~riak.datatypes.Datatype`, based on the presence and value @@ -216,6 +216,9 @@ def get(self, key, r=None, pr=None, timeout=None, include_context=None, :type basic_quorum: bool :param notfound_ok: whether to treat not-found responses as successful :type notfound_ok: bool + :param head_only: whether to fetch without value, so only metadata + (only available on PB transport) + :type head_only: bool :rtype: :class:`RiakObject ` or :class:`~riak.datatypes.Datatype` @@ -231,7 +234,8 @@ def get(self, key, r=None, pr=None, timeout=None, include_context=None, obj = RiakObject(self._client, self, key) return obj.reload(r=r, pr=pr, timeout=timeout, basic_quorum=basic_quorum, - notfound_ok=notfound_ok) + notfound_ok=notfound_ok, + head_only=head_only) def multiget(self, keys, r=None, pr=None, timeout=None, basic_quorum=None, notfound_ok=None): diff --git a/riak/client/operations.py b/riak/client/operations.py index c143cfbc..463a612c 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -680,7 +680,7 @@ def ts_stream_keys(self, table, timeout=None): @retryable def get(self, transport, robj, r=None, pr=None, timeout=None, - basic_quorum=None, notfound_ok=None): + basic_quorum=None, notfound_ok=None, head_only=False): """ get(robj, r=None, pr=None, timeout=None) @@ -702,6 +702,9 @@ def get(self, transport, robj, r=None, pr=None, timeout=None, :type basic_quorum: bool :param notfound_ok: whether to treat not-found responses as successful :type notfound_ok: bool + :param head_only: whether to fetch without value, so only metadata + (only available on PB transport) + :type head_only: bool """ _validate_timeout(timeout) if not isinstance(robj.key, six.string_types): @@ -710,7 +713,8 @@ def get(self, transport, robj, r=None, pr=None, timeout=None, return transport.get(robj, r=r, pr=pr, timeout=timeout, basic_quorum=basic_quorum, - notfound_ok=notfound_ok) + notfound_ok=notfound_ok, + head_only=head_only) @retryable def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, diff --git a/riak/codecs/pbuf.py b/riak/codecs/pbuf.py index 0bd792e5..777a5e13 100644 --- a/riak/codecs/pbuf.py +++ b/riak/codecs/pbuf.py @@ -892,7 +892,8 @@ def decode_preflist(self, item): return result def encode_get(self, robj, r=None, pr=None, timeout=None, - basic_quorum=None, notfound_ok=None): + basic_quorum=None, notfound_ok=None, + head_only=False): bucket = robj.bucket req = riak.pb.riak_kv_pb2.RpbGetReq() if r: @@ -911,6 +912,7 @@ def encode_get(self, robj, r=None, pr=None, timeout=None, req.bucket = str_to_bytes(bucket.name) self._add_bucket_type(req, bucket.bucket_type) req.key = str_to_bytes(robj.key) + req.head = head_only mc = riak.pb.messages.MSG_CODE_GET_REQ rc = riak.pb.messages.MSG_CODE_GET_RESP return Msg(mc, req.SerializeToString(), rc) diff --git a/riak/content.py b/riak/content.py index d885827b..6b3f080d 100644 --- a/riak/content.py +++ b/riak/content.py @@ -90,6 +90,8 @@ def _serialize(self, value): format(self.content_type)) def _deserialize(self, value): + if not value: + return value decoder = self._robject.bucket.get_decoder(self.content_type) if decoder: return decoder(value) diff --git a/riak/riak_object.py b/riak/riak_object.py index ab7dd375..7657e2a4 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -269,7 +269,7 @@ def store(self, w=None, dw=None, pw=None, return_body=True, return self def reload(self, r=None, pr=None, timeout=None, basic_quorum=None, - notfound_ok=None): + notfound_ok=None, head_only=False): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object @@ -293,10 +293,13 @@ def reload(self, r=None, pr=None, timeout=None, basic_quorum=None, :type basic_quorum: bool :param notfound_ok: whether to treat not-found responses as successful :type notfound_ok: bool + :param head_only: whether to fetch without value, so only metadata + (only available on PB transport) + :type head_only: bool :rtype: :class:`RiakObject` """ - self.client.get(self, r=r, pr=pr, timeout=timeout) + self.client.get(self, r=r, pr=pr, timeout=timeout, head_only=head_only) return self def delete(self, r=None, w=None, dw=None, pr=None, pw=None, diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 67dd901f..d0de3650 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -79,6 +79,13 @@ def test_no_returnbody(self): o = bucket.new(self.key_name, "bar").store(return_body=False) self.assertEqual(o.vclock, None) + def test_get_no_returnbody(self): + bucket = self.client.bucket(self.bucket_name) + o = bucket.new(self.key_name, "Ain't no body") + o.store() + stored_object = bucket.get(self.key_name, head_only=True) + self.assertEqual(stored_object.data, '') + def test_many_link_headers_should_work_fine(self): bucket = self.client.bucket(self.bucket_name) o = bucket.new("lots_of_links", "My god, it's full of links!") diff --git a/riak/transports/tcp/transport.py b/riak/transports/tcp/transport.py index 99415093..2a13e604 100644 --- a/riak/transports/tcp/transport.py +++ b/riak/transports/tcp/transport.py @@ -132,7 +132,7 @@ def _set_client_id(self, client_id): doc="""the client ID for this connection""") def get(self, robj, r=None, pr=None, timeout=None, basic_quorum=None, - notfound_ok=None): + notfound_ok=None, head_only=False): """ Serialize get request and deserialize response """ @@ -140,7 +140,7 @@ def get(self, robj, r=None, pr=None, timeout=None, basic_quorum=None, codec = self._get_codec(msg_code) msg = codec.encode_get(robj, r, pr, timeout, basic_quorum, - notfound_ok) + notfound_ok, head_only) resp_code, resp = self._request(msg, codec) return codec.decode_get(robj, resp) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index bda18e35..ba413865 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -54,7 +54,7 @@ def ping(self): raise NotImplementedError def get(self, robj, r=None, pr=None, timeout=None, basic_quorum=None, - notfound_ok=None): + notfound_ok=None, head_only=False): """ Fetches an object. """ From f124d7b8d973b7555d49997b45de42c1c8de343e Mon Sep 17 00:00:00 2001 From: Steffen Oschatz Date: Fri, 18 Nov 2016 14:36:36 +0100 Subject: [PATCH 2/5] prevent http test for pbc only --- riak/tests/test_kv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index d0de3650..3a966ade 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -8,7 +8,7 @@ from time import sleep from riak import ConflictError, RiakBucket, RiakError from riak.resolver import default_resolver, last_written_resolver -from riak.tests import RUN_KV, RUN_RESOLVE +from riak.tests import RUN_KV, RUN_RESOLVE, PROTOCOL from riak.tests.base import IntegrationTestBase from riak.tests.comparison import Comparison @@ -79,6 +79,7 @@ def test_no_returnbody(self): o = bucket.new(self.key_name, "bar").store(return_body=False) self.assertEqual(o.vclock, None) + @unittest.skipUnless(PROTOCOL == 'pbc', 'Only available on pbc') def test_get_no_returnbody(self): bucket = self.client.bucket(self.bucket_name) o = bucket.new(self.key_name, "Ain't no body") From 7be81998559111c1eabc1c7636ab2194cc87fa63 Mon Sep 17 00:00:00 2001 From: Steffen Oschatz Date: Fri, 18 Nov 2016 14:37:13 +0100 Subject: [PATCH 3/5] py3 combat --- riak/tests/test_kv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 3a966ade..ff2a0786 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -85,7 +85,7 @@ def test_get_no_returnbody(self): o = bucket.new(self.key_name, "Ain't no body") o.store() stored_object = bucket.get(self.key_name, head_only=True) - self.assertEqual(stored_object.data, '') + self.assertFalse(stored_object.data) def test_many_link_headers_should_work_fine(self): bucket = self.client.bucket(self.bucket_name) From bd66849eff9c3000bf1c2fd15a6319024d78afa7 Mon Sep 17 00:00:00 2001 From: Steffen Oschatz Date: Fri, 18 Nov 2016 15:02:28 +0100 Subject: [PATCH 4/5] added head parameter also to http to not break combat --- riak/transports/http/transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 1599d368..c2d7c6dc 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -107,7 +107,7 @@ def get_resources(self): return {} def get(self, robj, r=None, pr=None, timeout=None, basic_quorum=None, - notfound_ok=None): + notfound_ok=None, head_only=False): """ Get a bucket/key from the server """ From 9a3aa338f12a795a3779ca800133a6b74b5b5ff2 Mon Sep 17 00:00:00 2001 From: Steffen Oschatz Date: Fri, 18 Nov 2016 15:56:30 +0100 Subject: [PATCH 5/5] support head only request on multiget call --- riak/bucket.py | 9 +++++++-- riak/tests/test_kv.py | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index be499441..70340f08 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -238,7 +238,8 @@ def get(self, key, r=None, pr=None, timeout=None, include_context=None, head_only=head_only) def multiget(self, keys, r=None, pr=None, timeout=None, - basic_quorum=None, notfound_ok=None): + basic_quorum=None, notfound_ok=None, + head_only=False): """ Retrieves a list of keys belonging to this bucket in parallel. @@ -255,6 +256,9 @@ def multiget(self, keys, r=None, pr=None, timeout=None, :type basic_quorum: bool :param notfound_ok: whether to treat not-found responses as successful :type notfound_ok: bool + :param head_only: whether to fetch without value, so only metadata + (only available on PB transport) + :type head_only: bool :rtype: list of :class:`RiakObjects `, :class:`Datatypes `, or tuples of bucket_type, bucket, key, and the exception raised on fetch @@ -262,7 +266,8 @@ def multiget(self, keys, r=None, pr=None, timeout=None, bkeys = [(self.bucket_type.name, self.name, key) for key in keys] return self._client.multiget(bkeys, r=r, pr=pr, timeout=timeout, basic_quorum=basic_quorum, - notfound_ok=notfound_ok) + notfound_ok=notfound_ok, + head_only=head_only) def _get_resolver(self): if callable(self._resolver): diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index ff2a0786..7db30a08 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -84,9 +84,14 @@ def test_get_no_returnbody(self): bucket = self.client.bucket(self.bucket_name) o = bucket.new(self.key_name, "Ain't no body") o.store() + stored_object = bucket.get(self.key_name, head_only=True) self.assertFalse(stored_object.data) + list_of_objects = bucket.multiget([self.key_name], head_only=True) + for stored_object in list_of_objects: + self.assertFalse(stored_object.data) + def test_many_link_headers_should_work_fine(self): bucket = self.client.bucket(self.bucket_name) o = bucket.new("lots_of_links", "My god, it's full of links!")