Fix backup test hangs and teardown failures

Author: aseering

samples/samples/archived/backup_snippet.py

@@ -218,7 +218,8 @@ def delete_backup(instance_id, backup_id):
     backup.reload()
 
     # Wait for databases that reference this backup to finish optimizing.
-    while backup.referencing_databases:
+    timeout_at = time.time() + 600  # 10 minutes
+    while backup.referencing_databases and time.time() < timeout_at:
         time.sleep(30)
         backup.reload()
 

samples/samples/archived/backup_snippet_test.py

@@ -83,6 +83,7 @@ def test_copy_backup(capsys, instance_id, spanner_client):
     assert COPY_BACKUP_ID in out
 
 
+@pytest.mark.skip(reason="skipped since the KMS keys are not added on test project")
 @pytest.mark.dependency(name="create_backup_with_encryption_key")
 def test_create_backup_with_encryption_key(
     capsys,
@@ -115,12 +116,8 @@ def test_restore_database(capsys, instance_id, sample_database):
     assert BACKUP_ID in out
 
 
-@pytest.mark.skip(
-    reason="same test passes on unarchived test suite, "
-    "but fails here. Needs investigation"
-)
+@pytest.mark.skip(reason="skipped since the KMS keys are not added on test project")
 @pytest.mark.dependency(depends=["create_backup_with_encryption_key"])
-@RetryErrors(exception=DeadlineExceeded, max_tries=2)
 def test_restore_database_with_encryption_key(
     capsys,
     instance_id,

samples/samples/backup_sample.py

@@ -547,7 +547,8 @@ def delete_backup(instance_id, backup_id):
     )
 
     # Wait for databases that reference this backup to finish optimizing.
-    while backup.referencing_databases:
+    timeout_at = time.time() + 600  # 10 minutes
+    while backup.referencing_databases and time.time() < timeout_at:
         time.sleep(30)
         backup = database_admin_api.get_backup(
             backup_pb.GetBackupRequest(

samples/samples/backup_sample_test.py

@@ -85,6 +85,7 @@ def test_copy_backup(capsys, instance_id, spanner_client):
     assert COPY_BACKUP_ID in out
 
 
+@pytest.mark.skip(reason="skipped since the KMS keys are not added on test project")
 @pytest.mark.dependency(name="create_backup_with_encryption_key")
 def test_create_backup_with_encryption_key(
     capsys,
@@ -154,8 +155,8 @@ def test_restore_database(capsys, instance_id, sample_database):
     assert BACKUP_ID in out
 
 
+@pytest.mark.skip(reason="skipped since the KMS keys are not added on test project")
 @pytest.mark.dependency(depends=["create_backup_with_encryption_key"])
-@RetryErrors(exception=DeadlineExceeded, max_tries=2)
 def test_restore_database_with_encryption_key(
     capsys,
     instance_id,

samples/samples/conftest.py

@@ -274,6 +274,11 @@ def create_sample_database(
 
         yield sample_database
 
+        for backup_pb in sample_instance.list_backups():
+            backup_obj = backup.Backup.from_pb(backup_pb, sample_instance)
+            if backup_obj.database == sample_database.name:
+                backup_obj.delete()
+
         sample_database.drop()
         return
 
@@ -288,6 +293,11 @@ def create_sample_database(
 
     yield sample_database
 
+    for backup_pb in sample_instance.list_backups():
+        backup_obj = backup.Backup.from_pb(backup_pb, sample_instance)
+        if backup_obj.database == sample_database.name:
+            backup_obj.delete()
+
     sample_database.drop()