Skip to content

Commit c7b142d

Browse files
daniel-sanchedaniel-sanche
committed
improve system test cleanup
1 parent febdaac commit c7b142d

3 files changed

Lines changed: 113 additions & 72 deletions

File tree

‎packages/google-cloud-bigtable/tests/system/admin_overlay/test_system_async.py‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
generate_unique_suffix,
4040
)
4141

42+
from google.api_core import exceptions
4243
if CrossSync.is_async:
4344
from google.api_core import operation_async as api_core_operation
4445
else:
@@ -96,7 +97,10 @@ async def instances_to_delete(instance_admin_client):
9697
yield instances
9798
finally:
9899
for instance in instances:
99-
await instance_admin_client.delete_instance(name=instance.name)
100+
try:
101+
await instance_admin_client.delete_instance(name=instance.name)
102+
except exceptions.NotFound:
103+
pass
100104

101105

102106
@CrossSync.convert
@@ -108,7 +112,10 @@ async def backups_to_delete(table_admin_client):
108112
yield backups
109113
finally:
110114
for backup in backups:
111-
await table_admin_client.delete_backup(name=backup.name)
115+
try:
116+
await table_admin_client.delete_backup(name=backup.name)
117+
except exceptions.NotFound:
118+
pass
112119

113120

114121
@CrossSync.convert
@@ -159,9 +166,16 @@ async def create_instance(
159166
clusters=clusters,
160167
)
161168
operation = await instance_admin_client.create_instance(create_instance_request)
169+
170+
# add to cleanup list before waiting for result, in case of timeout
171+
instance_name = instance_admin_client.instance_path(project_id, instance_id)
172+
instances_to_delete.append(admin_v2.Instance(name=instance_name))
173+
162174
instance = await operation.result()
163175

164-
instances_to_delete.append(instance)
176+
# replace with full instance object
177+
instances_to_delete[-1] = instance
178+
165179

166180
# Create a table within the instance
167181
create_table_request = admin_v2.CreateTableRequest(
@@ -246,8 +260,14 @@ async def create_backup(
246260
)
247261
)
248262

263+
# add to cleanup list before waiting for result, in case of timeout
264+
backups_to_delete.append(admin_v2.Backup(name=f"{cluster_name}/backups/{backup_id}"))
265+
249266
backup = await operation.result()
250-
backups_to_delete.append(backup)
267+
268+
# replace with full backup object
269+
backups_to_delete[-1] = backup
270+
251271
return backup
252272

253273

‎packages/google-cloud-bigtable/tests/system/admin_overlay/test_system_autogen.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from typing import Tuple
2121

2222
import pytest
23+
from google.api_core import exceptions
2324
from google.api_core import operation as api_core_operation
2425
from google.cloud.environment_vars import BIGTABLE_EMULATOR
2526

@@ -79,7 +80,10 @@ def instances_to_delete(instance_admin_client):
7980
yield instances
8081
finally:
8182
for instance in instances:
82-
instance_admin_client.delete_instance(name=instance.name)
83+
try:
84+
instance_admin_client.delete_instance(name=instance.name)
85+
except exceptions.NotFound:
86+
pass
8387

8488

8589
@pytest.fixture(scope="session")
@@ -89,7 +93,10 @@ def backups_to_delete(table_admin_client):
8993
yield backups
9094
finally:
9195
for backup in backups:
92-
table_admin_client.delete_backup(name=backup.name)
96+
try:
97+
table_admin_client.delete_backup(name=backup.name)
98+
except exceptions.NotFound:
99+
pass
93100

94101

95102
def create_instance(
@@ -127,8 +134,10 @@ def create_instance(
127134
clusters=clusters,
128135
)
129136
operation = instance_admin_client.create_instance(create_instance_request)
137+
instance_name = instance_admin_client.instance_path(project_id, instance_id)
138+
instances_to_delete.append(admin_v2.Instance(name=instance_name))
130139
instance = operation.result()
131-
instances_to_delete.append(instance)
140+
instances_to_delete[-1] = instance
132141
create_table_request = admin_v2.CreateTableRequest(
133142
parent=instance_admin_client.instance_path(project_id, instance_id),
134143
table_id=TEST_TABLE_NAME,
@@ -189,8 +198,11 @@ def create_backup(
189198
),
190199
)
191200
)
201+
backups_to_delete.append(
202+
admin_v2.Backup(name=f"{cluster_name}/backups/{backup_id}")
203+
)
192204
backup = operation.result()
193-
backups_to_delete.append(backup)
205+
backups_to_delete[-1] = backup
194206
return backup
195207

196208

‎packages/google-cloud-bigtable/tests/system/data/__init__.py‎

Lines changed: 73 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -109,22 +109,27 @@ def instance_id(self, admin_client, project_id, cluster_config):
109109
yield instance_id
110110
else:
111111
try:
112-
operation = admin_client.instance_admin_client.create_instance(
113-
parent=f"projects/{project_id}",
114-
instance_id=instance_id,
115-
instance=types.Instance(
116-
display_name="Test Instance",
117-
# labels={"python-system-test": "true"},
118-
),
119-
clusters=cluster_config,
120-
)
121-
operation.result(timeout=240)
122-
except exceptions.AlreadyExists:
123-
pass
124-
yield instance_id
125-
admin_client.instance_admin_client.delete_instance(
126-
name=f"projects/{project_id}/instances/{instance_id}"
127-
)
112+
try:
113+
operation = admin_client.instance_admin_client.create_instance(
114+
parent=f"projects/{project_id}",
115+
instance_id=instance_id,
116+
instance=types.Instance(
117+
display_name="Test Instance",
118+
# labels={"python-system-test": "true"},
119+
),
120+
clusters=cluster_config,
121+
)
122+
operation.result(timeout=240)
123+
except exceptions.AlreadyExists:
124+
pass
125+
yield instance_id
126+
finally:
127+
try:
128+
admin_client.instance_admin_client.delete_instance(
129+
name=f"projects/{project_id}/instances/{instance_id}"
130+
)
131+
except exceptions.NotFound:
132+
pass
128133

129134
@pytest.fixture(scope="session")
130135
def column_split_config(self):
@@ -168,28 +173,30 @@ def table_id(
168173
retry = retry.Retry(
169174
predicate=retry.if_exception_type(exceptions.FailedPrecondition)
170175
)
176+
parent_path = f"projects/{project_id}/instances/{instance_id}"
171177
try:
172-
parent_path = f"projects/{project_id}/instances/{instance_id}"
173-
print(f"Creating table: {parent_path}/tables/{init_table_id}")
174-
admin_client.table_admin_client.create_table(
175-
request={
176-
"parent": parent_path,
177-
"table_id": init_table_id,
178-
"table": {"column_families": column_family_config},
179-
"initial_splits": [{"key": key} for key in column_split_config],
180-
},
181-
retry=retry,
182-
)
183-
except exceptions.AlreadyExists:
184-
pass
185-
yield init_table_id
186-
print(f"Deleting table: {parent_path}/tables/{init_table_id}")
187-
try:
188-
admin_client.table_admin_client.delete_table(
189-
name=f"{parent_path}/tables/{init_table_id}"
190-
)
191-
except exceptions.NotFound:
192-
print(f"Table {init_table_id} not found, skipping deletion")
178+
try:
179+
print(f"Creating table: {parent_path}/tables/{init_table_id}")
180+
admin_client.table_admin_client.create_table(
181+
request={
182+
"parent": parent_path,
183+
"table_id": init_table_id,
184+
"table": {"column_families": column_family_config},
185+
"initial_splits": [{"key": key} for key in column_split_config],
186+
},
187+
retry=retry,
188+
)
189+
except exceptions.AlreadyExists:
190+
pass
191+
yield init_table_id
192+
finally:
193+
print(f"Deleting table: {parent_path}/tables/{init_table_id}")
194+
try:
195+
admin_client.table_admin_client.delete_table(
196+
name=f"{parent_path}/tables/{init_table_id}"
197+
)
198+
except exceptions.NotFound:
199+
print(f"Table {init_table_id} not found, skipping deletion")
193200

194201
@pytest.fixture(scope="session")
195202
def authorized_view_id(
@@ -217,36 +224,38 @@ def authorized_view_id(
217224
parent_path = f"projects/{project_id}/instances/{instance_id}/tables/{table_id}"
218225
new_path = f"{parent_path}/authorizedViews/{new_view_id}"
219226
try:
220-
print(f"Creating view: {new_path}")
221-
admin_client.table_admin_client.create_authorized_view(
222-
request={
223-
"parent": parent_path,
224-
"authorized_view_id": new_view_id,
225-
"authorized_view": {
226-
"subset_view": {
227-
"row_prefixes": [ALLOW_ALL],
228-
"family_subsets": {
229-
TEST_FAMILY: ALL_QUALIFIERS,
230-
TEST_FAMILY_2: ALL_QUALIFIERS,
231-
TEST_AGGREGATE_FAMILY: ALL_QUALIFIERS,
227+
try:
228+
print(f"Creating view: {new_path}")
229+
admin_client.table_admin_client.create_authorized_view(
230+
request={
231+
"parent": parent_path,
232+
"authorized_view_id": new_view_id,
233+
"authorized_view": {
234+
"subset_view": {
235+
"row_prefixes": [ALLOW_ALL],
236+
"family_subsets": {
237+
TEST_FAMILY: ALL_QUALIFIERS,
238+
TEST_FAMILY_2: ALL_QUALIFIERS,
239+
TEST_AGGREGATE_FAMILY: ALL_QUALIFIERS,
240+
},
232241
},
233242
},
234243
},
235-
},
236-
retry=retry,
237-
)
238-
except exceptions.AlreadyExists:
239-
pass
240-
except exceptions.MethodNotImplemented:
241-
# will occur when run in emulator. Pass empty id
242-
new_view_id = None
243-
yield new_view_id
244-
if new_view_id:
245-
print(f"Deleting view: {new_path}")
246-
try:
247-
admin_client.table_admin_client.delete_authorized_view(name=new_path)
248-
except exceptions.NotFound:
249-
print(f"View {new_view_id} not found, skipping deletion")
244+
retry=retry,
245+
)
246+
except exceptions.AlreadyExists:
247+
pass
248+
except exceptions.MethodNotImplemented:
249+
# will occur when run in emulator. Pass empty id
250+
new_view_id = None
251+
yield new_view_id
252+
finally:
253+
if new_view_id:
254+
print(f"Deleting view: {new_path}")
255+
try:
256+
admin_client.table_admin_client.delete_authorized_view(name=new_path)
257+
except exceptions.NotFound:
258+
print(f"View {new_view_id} not found, skipping deletion")
250259

251260
@pytest.fixture(scope="session")
252261
def project_id(self, client):

0 commit comments

Comments
 (0)