Skip to content

Commit 6362669

Browse files
committed
PYCBC-1764 - Improve how client handles KV operations across the C-extension boundary
Motivation ---------- Improve KV operation throughput for v4.6.x when both observability functionality is enabled (default) or disabled. Changes ------- * Added pre-interned dict keys for the bindings to use when building C++/Python objects. * Added pycbc_kv_request type for KV operations to use. Allows request builders to create request object that will be used by bindings instead of intermediary data_class and dict objects. * Moved collection request builders to create pycbc_kv_request objects. * Updated bindings to only use native C++ objects when sending requests to the C++ core. This helps multi ops use C++ parallelism to a greater degree. * Multi ops in the bindings now build a batch of C++ requests, release the GIL to send all the request to the C++ core and collect results, then build all the Python objects (w/ the GIL re-acquired) after all the results have returned. * Added with_metrics flag to help guard against checking start/end time in multi ops when metrics are not enabled. * Moved kv_ops in the binding_map to be a NamedTuple. Each operation in the binding is now accessed via new KeyValueOperationCode and KeyValueMultiOperationCode IntEnums that align to the appropriate index in the NamedTuple. * Updated autogen tooling to handle new logic and types. * Update acouchbase & txcouchbase APIs to use new logic. Change-Id: I5a0a4abfc0921607ff9b571c9950935266125fb8 Reviewed-on: https://review.couchbase.org/c/couchbase-python-client/+/244054 Tested-by: Build Bot <build@couchbase.com> Reviewed-by: Dimitris Christodoulou <dimitris.christodoulou@couchbase.com>
1 parent f317a0f commit 6362669

52 files changed

Lines changed: 4589 additions & 4066 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

acouchbase/collection.py

Lines changed: 38 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -121,11 +121,11 @@ async def get(self,
121121
"""
122122
instruments = self._impl.observability_instruments
123123
if instruments.is_noop:
124-
req = self._impl.request_builder.build_get_request(key, None, *opts, **kwargs)
125-
return await self._impl.get(req, None)
124+
req, transcoder = self._impl.request_builder.build_get_request(key, None, *opts, **kwargs)
125+
return await self._impl.get(req, transcoder, None)
126126
async with ObservableRequestHandler(KeyValueOperationType.Get, instruments) as obs_handler:
127-
req = self._impl.request_builder.build_get_request(key, obs_handler, *opts, **kwargs)
128-
return await self._impl.get(req, obs_handler)
127+
req, transcoder = self._impl.request_builder.build_get_request(key, obs_handler, *opts, **kwargs)
128+
return await self._impl.get(req, transcoder, obs_handler)
129129

130130
async def get_any_replica(self,
131131
key, # type: str
@@ -175,11 +175,12 @@ async def get_any_replica(self,
175175
""" # noqa: E501
176176
instruments = self._impl.observability_instruments
177177
if instruments.is_noop:
178-
req = self._impl.request_builder.build_get_any_replica_request(key, None, *opts, **kwargs)
179-
return await self._impl.get_any_replica(req, None)
178+
req, transcoder = self._impl.request_builder.build_get_any_replica_request(key, None, *opts, **kwargs)
179+
return await self._impl.get_any_replica(req, transcoder, None)
180180
async with ObservableRequestHandler(KeyValueOperationType.GetAnyReplica, instruments) as obs_handler:
181-
req = self._impl.request_builder.build_get_any_replica_request(key, obs_handler, *opts, **kwargs)
182-
return await self._impl.get_any_replica(req, obs_handler)
181+
req, transcoder = self._impl.request_builder.build_get_any_replica_request(
182+
key, obs_handler, *opts, **kwargs)
183+
return await self._impl.get_any_replica(req, transcoder, obs_handler)
183184

184185
async def get_all_replicas(self,
185186
key, # type: str
@@ -249,11 +250,12 @@ async def get_all_replicas(self,
249250
"""
250251
instruments = self._impl.observability_instruments
251252
if instruments.is_noop:
252-
req = self._impl.request_builder.build_get_all_replicas_request(key, None, *opts, **kwargs)
253-
return await self._impl.get_all_replicas(req, None)
253+
req, transcoder = self._impl.request_builder.build_get_all_replicas_request(key, None, *opts, **kwargs)
254+
return await self._impl.get_all_replicas(req, transcoder, None)
254255
async with ObservableRequestHandler(KeyValueOperationType.GetAllReplicas, instruments) as obs_handler:
255-
req = self._impl.request_builder.build_get_all_replicas_request(key, obs_handler, *opts, **kwargs)
256-
return await self._impl.get_all_replicas(req, obs_handler)
256+
req, transcoder = self._impl.request_builder.build_get_all_replicas_request(
257+
key, obs_handler, *opts, **kwargs)
258+
return await self._impl.get_all_replicas(req, transcoder, obs_handler)
257259

258260
async def exists(self,
259261
key, # type: str
@@ -663,11 +665,12 @@ async def get_and_touch(self,
663665
"""
664666
instruments = self._impl.observability_instruments
665667
if instruments.is_noop:
666-
req = self._impl.request_builder.build_get_and_touch_request(key, expiry, None, *opts, **kwargs)
667-
return await self._impl.get_and_touch(req, None)
668+
req, transcoder = self._impl.request_builder.build_get_and_touch_request(key, expiry, None, *opts, **kwargs)
669+
return await self._impl.get_and_touch(req, transcoder, None)
668670
async with ObservableRequestHandler(KeyValueOperationType.GetAndTouch, instruments) as obs_handler:
669-
req = self._impl.request_builder.build_get_and_touch_request(key, expiry, obs_handler, *opts, **kwargs)
670-
return await self._impl.get_and_touch(req, obs_handler)
671+
req, transcoder = self._impl.request_builder.build_get_and_touch_request(
672+
key, expiry, obs_handler, *opts, **kwargs)
673+
return await self._impl.get_and_touch(req, transcoder, obs_handler)
671674

672675
async def get_and_lock(self,
673676
key, # type: str
@@ -724,11 +727,13 @@ async def get_and_lock(self,
724727
"""
725728
instruments = self._impl.observability_instruments
726729
if instruments.is_noop:
727-
req = self._impl.request_builder.build_get_and_lock_request(key, lock_time, None, *opts, **kwargs)
728-
return await self._impl.get_and_lock(req, None)
730+
req, transcoder = self._impl.request_builder.build_get_and_lock_request(
731+
key, lock_time, None, *opts, **kwargs)
732+
return await self._impl.get_and_lock(req, transcoder, None)
729733
async with ObservableRequestHandler(KeyValueOperationType.GetAndLock, instruments) as obs_handler:
730-
req = self._impl.request_builder.build_get_and_lock_request(key, lock_time, obs_handler, *opts, **kwargs)
731-
return await self._impl.get_and_lock(req, obs_handler)
734+
req, transcoder = self._impl.request_builder.build_get_and_lock_request(
735+
key, lock_time, obs_handler, *opts, **kwargs)
736+
return await self._impl.get_and_lock(req, transcoder, obs_handler)
732737

733738
async def unlock(self,
734739
key, # type: str
@@ -839,11 +844,12 @@ async def lookup_in(self,
839844
"""
840845
instruments = self._impl.observability_instruments
841846
if instruments.is_noop:
842-
req = self._impl.request_builder.build_lookup_in_request(key, spec, None, *opts, **kwargs)
843-
return await self._impl.lookup_in(req, None)
847+
req, transcoder = self._impl.request_builder.build_lookup_in_request(key, spec, None, *opts, **kwargs)
848+
return await self._impl.lookup_in(req, transcoder, None)
844849
async with ObservableRequestHandler(KeyValueOperationType.LookupIn, instruments) as obs_handler:
845-
req = self._impl.request_builder.build_lookup_in_request(key, spec, obs_handler, *opts, **kwargs)
846-
return await self._impl.lookup_in(req, obs_handler)
850+
req, transcoder = self._impl.request_builder.build_lookup_in_request(
851+
key, spec, obs_handler, *opts, **kwargs)
852+
return await self._impl.lookup_in(req, transcoder, obs_handler)
847853

848854
async def lookup_in_any_replica(self,
849855
key, # type: str
@@ -906,13 +912,13 @@ async def lookup_in_any_replica(self,
906912
"""
907913
instruments = self._impl.observability_instruments
908914
if instruments.is_noop:
909-
req = self._impl.request_builder.build_lookup_in_any_replica_request(
915+
req, transcoder = self._impl.request_builder.build_lookup_in_any_replica_request(
910916
key, spec, None, *opts, **kwargs)
911-
return await self._impl.lookup_in_any_replica(req, None)
917+
return await self._impl.lookup_in_any_replica(req, transcoder, None)
912918
async with ObservableRequestHandler(KeyValueOperationType.LookupInAnyReplica, instruments) as obs_handler:
913-
req = self._impl.request_builder.build_lookup_in_any_replica_request(
919+
req, transcoder = self._impl.request_builder.build_lookup_in_any_replica_request(
914920
key, spec, obs_handler, *opts, **kwargs)
915-
return await self._impl.lookup_in_any_replica(req, obs_handler)
921+
return await self._impl.lookup_in_any_replica(req, transcoder, obs_handler)
916922

917923
async def lookup_in_all_replicas(self,
918924
key, # type: str
@@ -998,13 +1004,13 @@ async def lookup_in_all_replicas(self,
9981004
""" # noqa: E501
9991005
instruments = self._impl.observability_instruments
10001006
if instruments.is_noop:
1001-
req = self._impl.request_builder.build_lookup_in_all_replicas_request(
1007+
req, transcoder = self._impl.request_builder.build_lookup_in_all_replicas_request(
10021008
key, spec, None, *opts, **kwargs)
1003-
return await self._impl.lookup_in_all_replicas(req, None)
1009+
return await self._impl.lookup_in_all_replicas(req, transcoder, None)
10041010
async with ObservableRequestHandler(KeyValueOperationType.LookupInAllReplicas, instruments) as obs_handler:
1005-
req = self._impl.request_builder.build_lookup_in_all_replicas_request(
1011+
req, transcoder = self._impl.request_builder.build_lookup_in_all_replicas_request(
10061012
key, spec, obs_handler, *opts, **kwargs)
1007-
return await self._impl.lookup_in_all_replicas(req, obs_handler)
1013+
return await self._impl.lookup_in_all_replicas(req, transcoder, obs_handler)
10081014

10091015
async def mutate_in(self,
10101016
key, # type: str

0 commit comments

Comments
 (0)