chore(tests): system tests for autogen API (#1151)

* tests: system tests for autogen API

* Fixed async system tests

* addressed review feedback

* Fixed system test failure at the end of a test run

* Linting

* more linting

Author: gkevinzheng

google/cloud/bigtable/admin_v2/overlay/__init__.py

@@ -30,7 +30,9 @@
 # client class under overlay/services.
 
 from .types import (
+    AsyncRestoreTableOperation,
     RestoreTableOperation,
+    WaitForConsistencyRequest,
 )
 
 from .services.bigtable_table_admin import (
@@ -39,7 +41,9 @@
 )
 
 __all__ = (
+    "AsyncRestoreTableOperation",
     "RestoreTableOperation",
     "BigtableTableAdminAsyncClient",
     "BigtableTableAdminClient",
+    "WaitForConsistencyRequest",
 )

google/cloud/bigtable/admin_v2/overlay/types/__init__.py

@@ -12,6 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .async_restore_table import (
+    AsyncRestoreTableOperation,
+)
+
 from .restore_table import (
     RestoreTableOperation,
 )
@@ -21,6 +25,7 @@
 )
 
 __all__ = (
+    "AsyncRestoreTableOperation",
     "RestoreTableOperation",
     "WaitForConsistencyRequest",
 )

google/cloud/bigtable/admin_v2/overlay/types/async_consistency.py

@@ -25,6 +25,10 @@
     OptionalRetry = Union[retries.Retry, object, None]  # type: ignore
 
 
+# The consistency check could take a very long time, so we wait indefinitely.
+DEFAULT_RETRY = async_future.DEFAULT_RETRY.with_timeout(None)
+
+
 class _AsyncCheckConsistencyPollingFuture(async_future.AsyncFuture):
     """A Future that polls an underlying `check_consistency` operation until it returns True.
 
@@ -48,7 +52,7 @@ class _AsyncCheckConsistencyPollingFuture(async_future.AsyncFuture):
             <google.cloud.bigtable.admin_v2.overlay.services.bigtable_table_admin.BigtableTableAdminClient.wait_for_consistency>`
             or :meth:`wait_for_replication
             <google.cloud.bigtable.admin_v2.overlay.services.bigtable_table_admin.BigtableTableAdminClient.wait_for_replication>`
-        retry (google.api_core.retry.Retry): The retry configuration used
+        retry (google.api_core.retry.AsyncRetry): The retry configuration used
             when polling. This can be used to control how often :meth:`done`
             is polled. Regardless of the retry's ``deadline``, it will be
             overridden by the ``timeout`` argument to :meth:`result`.
@@ -59,7 +63,7 @@ def __init__(
         check_consistency_call: Callable[
             [OptionalRetry], Awaitable[bigtable_table_admin.CheckConsistencyResponse]
         ],
-        retry: retries.Retry = async_future.DEFAULT_RETRY,
+        retry: retries.AsyncRetry = DEFAULT_RETRY,
         **kwargs
     ):
         super(_AsyncCheckConsistencyPollingFuture, self).__init__(retry=retry, **kwargs)

google/cloud/bigtable/admin_v2/overlay/types/consistency.py

@@ -25,6 +25,10 @@
     OptionalRetry = Union[retries.Retry, object, None]  # type: ignore
 
 
+# The consistency check could take a very long time, so we wait indefinitely.
+DEFAULT_RETRY = polling.DEFAULT_POLLING.with_timeout(None)
+
+
 class _CheckConsistencyPollingFuture(polling.PollingFuture):
     """A Future that polls an underlying `check_consistency` operation until it returns True.
 
@@ -55,7 +59,7 @@ def __init__(
         check_consistency_call: Callable[
             [OptionalRetry], bigtable_table_admin.CheckConsistencyResponse
         ],
-        polling: retries.Retry = polling.DEFAULT_POLLING,
+        polling: retries.Retry = DEFAULT_RETRY,
         **kwargs
     ):
         super(_CheckConsistencyPollingFuture, self).__init__(polling=polling, **kwargs)

setup.py

@@ -37,7 +37,7 @@
 # 'Development Status :: 5 - Production/Stable'
 release_status = "Development Status :: 5 - Production/Stable"
 dependencies = [
-    "google-api-core[grpc] >= 2.16.0, <3.0.0",
+    "google-api-core[grpc] >= 2.17.0, <3.0.0",
     "google-cloud-core >= 1.4.4, <3.0.0",
     "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0",
     "grpc-google-iam-v1 >= 0.12.4, <1.0.0",

testing/constraints-3.7.txt

@@ -5,7 +5,7 @@
 #
 # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev",
 # Then this file should have foo==1.14.0
-google-api-core==2.16.0
+google-api-core==2.17.0
 google-auth==2.14.1
 google-cloud-core==2.0.0
 grpc-google-iam-v1==0.12.4

testing/constraints-3.8.txt

@@ -5,7 +5,7 @@
 #
 # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev",
 # Then this file should have foo==1.14.0
-google-api-core==2.16.0
+google-api-core==2.17.0
 google-auth==2.14.1
 google-cloud-core==2.0.0
 grpc-google-iam-v1==0.12.4

tests/system/admin_overlay/__init__.py

@@ -0,0 +1,36 @@
+import google.auth
+
+import os
+import pytest
+import uuid
+
+
+INSTANCE_PREFIX = "admin-overlay-instance"
+BACKUP_PREFIX = "admin-overlay-backup"
+ROW_PREFIX = "test-row"
+
+DEFAULT_CLUSTER_LOCATIONS = ["us-east1-b"]
+REPLICATION_CLUSTER_LOCATIONS = ["us-east1-b", "us-west1-b"]
+TEST_TABLE_NAME = "system-test-table"
+TEST_BACKUP_TABLE_NAME = "system-test-backup-table"
+TEST_COLUMMN_FAMILY_NAME = "test-column"
+TEST_COLUMN_NAME = "value"
+NUM_ROWS = 500
+INITIAL_CELL_VALUE = "Hello"
+NEW_CELL_VALUE = "World"
+
+
+@pytest.fixture(scope="session")
+def admin_overlay_project_id():
+    _, default_project = google.auth.default()
+    yield os.getenv("GOOGLE_CLOUD_PROJECT") or default_project
+
+
+def generate_unique_suffix(name):
+    """
+    Generates a unique suffix for the name.
+
+    Uses UUID4 because using time.time doesn't guarantee
+    uniqueness when the time is frozen in containers.
+    """
+    return f"{name}-{uuid.uuid4().hex[:7]}"