Merge branch 'main' into csm_1_data_model

Author: daniel-sanche

tests/system/admin_overlay/test_system_async.py

@@ -143,20 +143,27 @@ async def create_instance(
             default_storage_type=storage_type,
         )
 
-    create_instance_request = admin_v2.CreateInstanceRequest(
-        parent=instance_admin_client.common_project_path(project_id),
-        instance_id=instance_id,
-        instance=admin_v2.Instance(
-            display_name=instance_id[
-                :30
-            ],  # truncate to 30 characters because of character limit
-        ),
-        clusters=clusters,
-    )
-    operation = await instance_admin_client.create_instance(create_instance_request)
-    instance = await operation.result()
+    # Instance and cluster creation are currently unsupported in the Bigtable emulator
+    if os.getenv(BIGTABLE_EMULATOR):
+        # All we need for system tests so far is the instance name.
+        instance = admin_v2.Instance(
+            name=instance_admin_client.instance_path(project_id, instance_id),
+        )
+    else:
+        create_instance_request = admin_v2.CreateInstanceRequest(
+            parent=instance_admin_client.common_project_path(project_id),
+            instance_id=instance_id,
+            instance=admin_v2.Instance(
+                display_name=instance_id[
+                    :30
+                ],  # truncate to 30 characters because of character limit
+            ),
+            clusters=clusters,
+        )
+        operation = await instance_admin_client.create_instance(create_instance_request)
+        instance = await operation.result()
 
-    instances_to_delete.append(instance)
+        instances_to_delete.append(instance)
 
     # Create a table within the instance
     create_table_request = admin_v2.CreateTableRequest(
@@ -272,6 +279,10 @@ async def assert_table_cell_value_equal_to(
     }
 )
 @CrossSync.pytest
+@pytest.mark.skipif(
+    os.getenv(BIGTABLE_EMULATOR),
+    reason="Backups are not supported in the Bigtable emulator",
+)
 @pytest.mark.parametrize(
     "second_instance_storage_type,expect_optimize_operation",
     [

tests/system/admin_overlay/test_system_autogen.py

@@ -113,15 +113,20 @@ def create_instance(
             location=instance_admin_client.common_location_path(project_id, location),
             default_storage_type=storage_type,
         )
-    create_instance_request = admin_v2.CreateInstanceRequest(
-        parent=instance_admin_client.common_project_path(project_id),
-        instance_id=instance_id,
-        instance=admin_v2.Instance(display_name=instance_id[:30]),
-        clusters=clusters,
-    )
-    operation = instance_admin_client.create_instance(create_instance_request)
-    instance = operation.result()
-    instances_to_delete.append(instance)
+    if os.getenv(BIGTABLE_EMULATOR):
+        instance = admin_v2.Instance(
+            name=instance_admin_client.instance_path(project_id, instance_id)
+        )
+    else:
+        create_instance_request = admin_v2.CreateInstanceRequest(
+            parent=instance_admin_client.common_project_path(project_id),
+            instance_id=instance_id,
+            instance=admin_v2.Instance(display_name=instance_id[:30]),
+            clusters=clusters,
+        )
+        operation = instance_admin_client.create_instance(create_instance_request)
+        instance = operation.result()
+        instances_to_delete.append(instance)
     create_table_request = admin_v2.CreateTableRequest(
         parent=instance_admin_client.instance_path(project_id, instance_id),
         table_id=TEST_TABLE_NAME,
@@ -201,6 +206,10 @@ def assert_table_cell_value_equal_to(
         assert latest_cell.value.decode("utf-8") == value
 
 
+@pytest.mark.skipif(
+    os.getenv(BIGTABLE_EMULATOR),
+    reason="Backups are not supported in the Bigtable emulator",
+)
 @pytest.mark.parametrize(
     "second_instance_storage_type,expect_optimize_operation",
     [(admin_v2.StorageType.HDD, False), (admin_v2.StorageType.SSD, True)],
@@ -215,15 +224,15 @@ def test_optimize_restored_table(
     second_instance_storage_type,
     expect_optimize_operation,
 ):
-    instance_with_backup, table_to_backup = create_instance(
+    (instance_with_backup, table_to_backup) = create_instance(
         instance_admin_client,
         table_admin_client,
         data_client,
         admin_overlay_project_id,
         instances_to_delete,
         admin_v2.StorageType.HDD,
     )
-    instance_to_restore, _ = create_instance(
+    (instance_to_restore, _) = create_instance(
         instance_admin_client,
         table_admin_client,
         data_client,
@@ -273,7 +282,7 @@ def test_wait_for_consistency(
     instances_to_delete,
     admin_overlay_project_id,
 ):
-    instance, table = create_instance(
+    (instance, table) = create_instance(
         instance_admin_client,
         table_admin_client,
         data_client,

tests/system/data/test_system_async.py

@@ -119,11 +119,14 @@ async def delete_rows(self):
 
 @CrossSync.convert_class(sync_name="TestSystem")
 class TestSystemAsync:
+    def _make_client(self):
+        project = os.getenv("GOOGLE_CLOUD_PROJECT") or None
+        return CrossSync.DataClient(project=project)
+
     @CrossSync.convert
     @CrossSync.pytest_fixture(scope="session")
     async def client(self):
-        project = os.getenv("GOOGLE_CLOUD_PROJECT") or None
-        async with CrossSync.DataClient(project=project) as client:
+        async with self._make_client() as client:
             yield client
 
     @CrossSync.convert
@@ -268,8 +271,7 @@ async def test_channel_refresh(self, table_id, instance_id, temp_rows):
         """
         await temp_rows.add_row(b"row_key_1")
         await temp_rows.add_row(b"row_key_2")
-        project = os.getenv("GOOGLE_CLOUD_PROJECT") or None
-        client = CrossSync.DataClient(project=project)
+        client = self._make_client()
         # start custom refresh task
         try:
             client._channel_refresh_task = CrossSync.create_task(

tests/system/data/test_system_autogen.py

@@ -102,10 +102,13 @@ def delete_rows(self):
 
 
 class TestSystem:
+    def _make_client(self):
+        project = os.getenv("GOOGLE_CLOUD_PROJECT") or None
+        return CrossSync._Sync_Impl.DataClient(project=project)
+
     @pytest.fixture(scope="session")
     def client(self):
-        project = os.getenv("GOOGLE_CLOUD_PROJECT") or None
-        with CrossSync._Sync_Impl.DataClient(project=project) as client:
+        with self._make_client() as client:
             yield client
 
     @pytest.fixture(scope="session", params=TARGETS)
@@ -222,8 +225,7 @@ def test_channel_refresh(self, table_id, instance_id, temp_rows):
         to ensure new channel works"""
         temp_rows.add_row(b"row_key_1")
         temp_rows.add_row(b"row_key_2")
-        project = os.getenv("GOOGLE_CLOUD_PROJECT") or None
-        client = CrossSync._Sync_Impl.DataClient(project=project)
+        client = self._make_client()
         try:
             client._channel_refresh_task = CrossSync._Sync_Impl.create_task(
                 client._manage_channel,
@@ -258,7 +260,7 @@ def test_mutation_set_cell(self, target, temp_rows):
         """Ensure cells can be set properly"""
         row_key = b"bulk_mutate"
         new_value = uuid.uuid4().hex.encode()
-        row_key, mutation = self._create_row_and_mutation(
+        (row_key, mutation) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value
         )
         target.mutate_row(row_key, mutation)
@@ -312,7 +314,7 @@ def test_bulk_mutations_set_cell(self, client, target, temp_rows):
         from google.cloud.bigtable.data.mutations import RowMutationEntry
 
         new_value = uuid.uuid4().hex.encode()
-        row_key, mutation = self._create_row_and_mutation(
+        (row_key, mutation) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value
         )
         bulk_mutation = RowMutationEntry(row_key, [mutation])
@@ -347,11 +349,11 @@ def test_mutations_batcher_context_manager(self, client, target, temp_rows):
         """test batcher with context manager. Should flush on exit"""
         from google.cloud.bigtable.data.mutations import RowMutationEntry
 
-        new_value, new_value2 = [uuid.uuid4().hex.encode() for _ in range(2)]
-        row_key, mutation = self._create_row_and_mutation(
+        (new_value, new_value2) = [uuid.uuid4().hex.encode() for _ in range(2)]
+        (row_key, mutation) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value
         )
-        row_key2, mutation2 = self._create_row_and_mutation(
+        (row_key2, mutation2) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value2
         )
         bulk_mutation = RowMutationEntry(row_key, [mutation])
@@ -372,7 +374,7 @@ def test_mutations_batcher_timer_flush(self, client, target, temp_rows):
         from google.cloud.bigtable.data.mutations import RowMutationEntry
 
         new_value = uuid.uuid4().hex.encode()
-        row_key, mutation = self._create_row_and_mutation(
+        (row_key, mutation) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value
         )
         bulk_mutation = RowMutationEntry(row_key, [mutation])
@@ -394,12 +396,12 @@ def test_mutations_batcher_count_flush(self, client, target, temp_rows):
         """batch should flush after flush_limit_mutation_count mutations"""
         from google.cloud.bigtable.data.mutations import RowMutationEntry
 
-        new_value, new_value2 = [uuid.uuid4().hex.encode() for _ in range(2)]
-        row_key, mutation = self._create_row_and_mutation(
+        (new_value, new_value2) = [uuid.uuid4().hex.encode() for _ in range(2)]
+        (row_key, mutation) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value
         )
         bulk_mutation = RowMutationEntry(row_key, [mutation])
-        row_key2, mutation2 = self._create_row_and_mutation(
+        (row_key2, mutation2) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value2
         )
         bulk_mutation2 = RowMutationEntry(row_key2, [mutation2])
@@ -426,12 +428,12 @@ def test_mutations_batcher_bytes_flush(self, client, target, temp_rows):
         """batch should flush after flush_limit_bytes bytes"""
         from google.cloud.bigtable.data.mutations import RowMutationEntry
 
-        new_value, new_value2 = [uuid.uuid4().hex.encode() for _ in range(2)]
-        row_key, mutation = self._create_row_and_mutation(
+        (new_value, new_value2) = [uuid.uuid4().hex.encode() for _ in range(2)]
+        (row_key, mutation) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value
         )
         bulk_mutation = RowMutationEntry(row_key, [mutation])
-        row_key2, mutation2 = self._create_row_and_mutation(
+        (row_key2, mutation2) = self._create_row_and_mutation(
             target, temp_rows, new_value=new_value2
         )
         bulk_mutation2 = RowMutationEntry(row_key2, [mutation2])
@@ -457,11 +459,11 @@ def test_mutations_batcher_no_flush(self, client, target, temp_rows):
 
         new_value = uuid.uuid4().hex.encode()
         start_value = b"unchanged"
-        row_key, mutation = self._create_row_and_mutation(
+        (row_key, mutation) = self._create_row_and_mutation(
             target, temp_rows, start_value=start_value, new_value=new_value
         )
         bulk_mutation = RowMutationEntry(row_key, [mutation])
-        row_key2, mutation2 = self._create_row_and_mutation(
+        (row_key2, mutation2) = self._create_row_and_mutation(
             target, temp_rows, start_value=start_value, new_value=new_value
         )
         bulk_mutation2 = RowMutationEntry(row_key2, [mutation2])

tests/unit/data/_sync_autogen/test_client.py

@@ -938,6 +938,9 @@ def _make_one(
 
     def test_ctor(self):
         from google.cloud.bigtable.data._helpers import _WarmedInstanceKey
+        from google.cloud.bigtable.data._metrics import (
+            BigtableClientSideMetricsController,
+        )
 
         expected_table_id = "table-id"
         expected_instance_id = "instance-id"
@@ -978,6 +981,8 @@ def test_ctor(self):
         instance_key = _WarmedInstanceKey(table.instance_name, table.app_profile_id)
         assert instance_key in client._active_instances
         assert client._instance_owners[instance_key] == {id(table)}
+        assert isinstance(table._metrics, BigtableClientSideMetricsController)
+        assert table._metrics.interceptor == client._metrics_interceptor
         assert table.default_operation_timeout == expected_operation_timeout
         assert table.default_attempt_timeout == expected_attempt_timeout
         assert (
@@ -1195,6 +1200,9 @@ def _make_one(
 
     def test_ctor(self):
         from google.cloud.bigtable.data._helpers import _WarmedInstanceKey
+        from google.cloud.bigtable.data._metrics import (
+            BigtableClientSideMetricsController,
+        )
 
         expected_table_id = "table-id"
         expected_instance_id = "instance-id"
@@ -1242,6 +1250,8 @@ def test_ctor(self):
         instance_key = _WarmedInstanceKey(view.instance_name, view.app_profile_id)
         assert instance_key in client._active_instances
         assert client._instance_owners[instance_key] == {id(view)}
+        assert isinstance(view._metrics, BigtableClientSideMetricsController)
+        assert view._metrics.interceptor == client._metrics_interceptor
         assert view.default_operation_timeout == expected_operation_timeout
         assert view.default_attempt_timeout == expected_attempt_timeout
         assert (