fix: support sub-meanings for datastore v2.20.3

google/cloud/ndb/model.py

@@ -2698,14 +2698,26 @@ def _from_datastore(self, ds_entity, value):
         Need to check the ds_entity for a compressed meaning that would
         indicate we are getting a compressed value.
         """
-        if self._name in ds_entity._meanings:
-            meaning = ds_entity._meanings[self._name][0]
-            if meaning == _MEANING_COMPRESSED and not self._compressed:
-                if self._repeated:
-                    for sub_value in value:
-                        sub_value.b_val = zlib.decompress(sub_value.b_val)
-                else:
-                    value.b_val = zlib.decompress(value.b_val)
+        if self._name in ds_entity._meanings and not self._compressed:
+            root_meaning = ds_entity._meanings[self._name][0]
+            sub_meanings = None
+            # meaning may be a tuple. Attempt unwrap
+            if isinstance(root_meaning, tuple):
+                root_meaning, sub_meanings = root_meaning
+            # decompress values if needed
+            if root_meaning == _MEANING_COMPRESSED and not self._repeated:
+                value.b_val = zlib.decompress(value.b_val)
+            elif root_meaning == _MEANING_COMPRESSED and self._repeated:
+                for sub_value in value:
+                    sub_value.b_val = zlib.decompress(sub_value.b_val)
+            elif isinstance(sub_meanings, list) and self._repeated:
+                for idx, sub_value in enumerate(value):
+                    try:
+                        if sub_meanings[idx] == _MEANING_COMPRESSED:
+                            sub_value.b_val = zlib.decompress(sub_value.b_val)
+                    except IndexError:
+                        # value list size exceeds sub_meanings list
+                        break
         return value
 
     def _db_set_compressed_meaning(self, p):

tests/unit/test_model.py

@@ -1929,9 +1929,12 @@ class ThisKind(model.Model):
         ds_entity = model._entity_to_ds_entity(entity)
         assert ds_entity["foo"] == compressed_value
 
-    @staticmethod
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.1 and lower",
+    )
     @pytest.mark.usefixtures("in_context")
-    def test__from_datastore_compressed_repeated_to_compressed():
+    def test__from_datastore_compressed_repeated_to_compressed_legacy(self):
         class ThisKind(model.Model):
             foo = model.BlobProperty(compressed=True, repeated=True)
 
@@ -1955,9 +1958,50 @@ class ThisKind(model.Model):
         ds_entity = model._entity_to_ds_entity(entity)
         assert ds_entity["foo"] == [compressed_value_one, compressed_value_two]
 
-    @staticmethod
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.2 and later",
+    )
+    @pytest.mark.parametrize(
+        "meaning",
+        [
+            (model._MEANING_COMPRESSED, None),  # set root meaning
+            (model._MEANING_COMPRESSED, []),
+            (model._MEANING_COMPRESSED, [1, 1]),
+            (None, [model._MEANING_COMPRESSED] * 2),  # set sub-meanings
+        ],
+    )
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_compressed_repeated_to_compressed(self, meaning):
+        class ThisKind(model.Model):
+            foo = model.BlobProperty(compressed=True, repeated=True)
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [compressed_value_one, compressed_value_two]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                meaning,
+                compressed_value,
+            )
+        }
+        datastore_entity._meanings = meanings
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        ds_entity = model._entity_to_ds_entity(entity)
+        assert ds_entity["foo"] == [compressed_value_one, compressed_value_two]
+
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.1 and lower",
+    )
     @pytest.mark.usefixtures("in_context")
-    def test__from_datastore_compressed_repeated_to_uncompressed():
+    def test__from_datastore_compressed_repeated_to_uncompressed_legacy(self):
         class ThisKind(model.Model):
             foo = model.BlobProperty(compressed=False, repeated=True)
 
@@ -1981,6 +2025,133 @@ class ThisKind(model.Model):
         ds_entity = model._entity_to_ds_entity(entity)
         assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two]
 
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.2 and later",
+    )
+    @pytest.mark.parametrize(
+        "meaning",
+        [
+            (model._MEANING_COMPRESSED, None),  # set root meaning
+            (model._MEANING_COMPRESSED, []),
+            (model._MEANING_COMPRESSED, [1, 1]),
+            (None, [model._MEANING_COMPRESSED] * 2),  # set sub-meanings
+        ],
+    )
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_compressed_repeated_to_uncompressed(self, meaning):
+        class ThisKind(model.Model):
+            foo = model.BlobProperty(compressed=False, repeated=True)
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [compressed_value_one, compressed_value_two]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                meaning,
+                compressed_value,
+            )
+        }
+        datastore_entity._meanings = meanings
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        ds_entity = model._entity_to_ds_entity(entity)
+        assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two]
+
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.2 and later",
+    )
+    @pytest.mark.parametrize(
+        "meaning",
+        [
+            (None, [model._MEANING_COMPRESSED, None]),
+            (None, [model._MEANING_COMPRESSED, None, None]),
+            (1, [model._MEANING_COMPRESSED, 1]),
+            (None, [model._MEANING_COMPRESSED]),
+        ],
+    )
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_compressed_repeated_to_uncompressed_mixed_meaning(
+        self, meaning
+    ):
+        """
+        One item is compressed, one uncompressed
+        """
+
+        class ThisKind(model.Model):
+            foo = model.BlobProperty(compressed=False, repeated=True)
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [compressed_value_one, compressed_value_two]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                meaning,
+                compressed_value,
+            )
+        }
+        datastore_entity._meanings = meanings
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        ds_entity = model._entity_to_ds_entity(entity)
+        assert ds_entity["foo"] == [uncompressed_value_one, compressed_value_two]
+
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.2 and later",
+    )
+    @pytest.mark.parametrize(
+        "meaning",
+        [
+            (None, None),
+            (None, []),
+            (None, [None]),
+            (None, [None, None]),
+            (1, []),
+            (1, [1]),
+            (1, [1, 1]),
+        ],
+    )
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_compressed_repeated_no_meaning(self, meaning):
+        """
+        could be uncompressed, but meaning not set
+        """
+
+        class ThisKind(model.Model):
+            foo = model.BlobProperty(compressed=False, repeated=True)
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [compressed_value_one, compressed_value_two]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                meaning,
+                compressed_value,
+            )
+        }
+        datastore_entity._meanings = meanings
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        ds_entity = model._entity_to_ds_entity(entity)
+        assert ds_entity["foo"] == [compressed_value_one, compressed_value_two]
+
     @staticmethod
     @pytest.mark.usefixtures("in_context")
     def test__from_datastore_uncompressed_to_uncompressed():