ci: add auto test cleanup for hosted_extractors.destinations (#2641)

Author: haakonvt

tests/tests_integration/test_api/test_hosted_extractors/conftest.py

@@ -21,18 +21,41 @@
 UPDATE_SOURCE_PREFIX = "to-update-"
 TEST_SOURCE_PREFIXES = (HUB_SOURCE_PREFIX, MQTT_SOURCE_PREFIX, UPDATE_SOURCE_PREFIX)
 
+DESTINATION_PREFIX = "myNewDestination-"
+DESTINATION_FOR_TESTING_PREFIX = "myNewDestinationForTesting-"
+UPDATE_DESTINATION_PREFIX = "toupdate-"
+TEST_DESTINATION_PREFIXES = (DESTINATION_PREFIX, DESTINATION_FOR_TESTING_PREFIX, UPDATE_DESTINATION_PREFIX)
+
+# Sources and Destinatinos are repeat offenders for hitting max limits because they are
+# quite low. We add specific cleanup for anything older than...:
+DELETE_THRESHOLD = 1 * 60 * 60 * 1000  # 1 hour in ms
+NOW_MS = int(time.time() * 1000)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def cleanup_stale_test_destinations(cognite_client: CogniteClient) -> None:
+    # max is low number like e.g. 50 per project, safe to list all:
+    all_destinations = cognite_client.hosted_extractors.destinations.list(limit=None)
+    stale = [
+        d.external_id
+        for d in all_destinations
+        if d.external_id
+        and d.external_id.startswith(TEST_DESTINATION_PREFIXES)
+        and NOW_MS - d.created_time >= DELETE_THRESHOLD
+    ]
+    if stale:
+        cognite_client.hosted_extractors.destinations.delete(stale, force=True, ignore_unknown_ids=True)
+
 
 @pytest.fixture(scope="session", autouse=True)
 def cleanup_stale_test_sources(cognite_client: CogniteClient) -> None:
     all_sources = cognite_client.hosted_extractors.sources.list(limit=None)
-    now_ms = int(time.time() * 1000)
-    # Clean up anyting older than 3 hours:
     stale = [
         s.external_id
         for s in all_sources
         if s.external_id
         and s.external_id.startswith(TEST_SOURCE_PREFIXES)
-        and now_ms - s.created_time >= 3 * 60 * 60 * 1000  # type: ignore [attr-defined]
+        and NOW_MS - s.created_time >= DELETE_THRESHOLD  # type: ignore [attr-defined]
     ]
     if stale:
         cognite_client.hosted_extractors.sources.delete(stale, force=True, ignore_unknown_ids=True)
@@ -58,7 +81,7 @@ def a_data_set(cognite_client: CogniteClient) -> DataSet:
 @pytest.fixture
 def one_destination(cognite_client: CogniteClient, fresh_session: SessionWrite) -> Iterator[Destination]:
     my_dest = DestinationWrite(
-        external_id=f"myNewDestination-{random_string(10)}",
+        external_id=f"{DESTINATION_PREFIX}{random_string(10)}",
         credentials=fresh_session,
     )
     created = cognite_client.hosted_extractors.destinations.create(my_dest)

tests/tests_integration/test_api/test_hosted_extractors/test_destinations.py

@@ -13,14 +13,18 @@
 )
 from cognite.client.exceptions import CogniteAPIError
 from cognite.client.utils._text import random_string
+from tests.tests_integration.test_api.test_hosted_extractors.conftest import (
+    DESTINATION_FOR_TESTING_PREFIX,
+    UPDATE_DESTINATION_PREFIX,
+)
 
 
 class TestDestinations:
     def test_create_update_retrieve_delete(
         self, cognite_client: CogniteClient, fresh_session: SessionWrite, a_data_set: DataSet
     ) -> None:
         my_dest = DestinationWrite(
-            external_id=f"myNewDestinationForTesting-{random_string(10)}",
+            external_id=f"{DESTINATION_FOR_TESTING_PREFIX}{random_string(10)}",
             credentials=fresh_session,
         )
         created: Destination | None = None
@@ -56,7 +60,7 @@ def test_update_using_write_object(
         self, cognite_client: CogniteClient, fresh_session: SessionWrite, a_data_set: DataSet
     ) -> None:
         my_dest = DestinationWrite(
-            external_id=f"toupdate-{random_string(10)}",
+            external_id=f"{UPDATE_DESTINATION_PREFIX}{random_string(10)}",
             credentials=fresh_session,
         )