Adding terraform trigger for cleanup script and spacing out onspot tests (GoogleCloudPlatform#5149)

Author: simrankaurb

tools/cloud-build/provision/README.md

@@ -36,6 +36,7 @@ When prompted for project, use integration test project.
 |------|--------|---------|
 | <a name="module_daily_image_test_runner_schedule"></a> [daily\_image\_test\_runner\_schedule](#module\_daily\_image\_test\_runner\_schedule) | ./trigger-schedule | n/a |
 | <a name="module_daily_project_cleanup_filestore_schedule"></a> [daily\_project\_cleanup\_filestore\_schedule](#module\_daily\_project\_cleanup\_filestore\_schedule) | ./trigger-schedule | n/a |
+| <a name="module_daily_project_cleanup_schedule"></a> [daily\_project\_cleanup\_schedule](#module\_daily\_project\_cleanup\_schedule) | ./trigger-schedule | n/a |
 | <a name="module_daily_project_cleanup_slurm_schedule"></a> [daily\_project\_cleanup\_slurm\_schedule](#module\_daily\_project\_cleanup\_slurm\_schedule) | ./trigger-schedule | n/a |
 | <a name="module_daily_test_schedule"></a> [daily\_test\_schedule](#module\_daily\_test\_schedule) | ./trigger-schedule | n/a |
 | <a name="module_weekly_build_dependency_check_schedule"></a> [weekly\_build\_dependency\_check\_schedule](#module\_weekly\_build\_dependency\_check\_schedule) | ./trigger-schedule | n/a |
@@ -44,6 +45,7 @@ When prompted for project, use integration test project.
 
 | Name | Type |
 |------|------|
+| [google_cloudbuild_trigger.daily_project_cleanup](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource |
 | [google_cloudbuild_trigger.daily_project_cleanup_filestore](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource |
 | [google_cloudbuild_trigger.daily_project_cleanup_slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource |
 | [google_cloudbuild_trigger.daily_test](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource |

tools/cloud-build/provision/daily-cleanup.tf

@@ -63,3 +63,29 @@ module "daily_project_cleanup_slurm_schedule" {
   schedule    = "0 0 * * MON-FRI"
   retry_count = 4
 }
+
+resource "google_cloudbuild_trigger" "daily_project_cleanup" {
+  name        = "DAILY-project-cleanup"
+  description = "A resource cleanup script to run periodically"
+  tags        = [local.notify_chat_tag]
+
+  git_file_source {
+    path      = "tools/cloud-build/project-cleanup.yaml"
+    revision  = local.ref_develop
+    uri       = var.repo_uri
+    repo_type = "GITHUB"
+  }
+
+  source_to_build {
+    uri       = var.repo_uri
+    ref       = local.ref_develop
+    repo_type = "GITHUB"
+  }
+}
+
+module "daily_project_cleanup_schedule" {
+  source      = "./trigger-schedule"
+  trigger     = google_cloudbuild_trigger.daily_project_cleanup
+  schedule    = "0 22 * * *"
+  retry_count = 4
+}

tools/cloud-build/provision/list_tests.py

@@ -54,9 +54,12 @@
     # (set_of_tests, min_distance)
     ((
         "ml-a4-highgpu-slurm",
-        "ml-a4-highgpu-onspot-slurm",
         "gke-a4"
     ), 2*60),
+    ((
+        "ml-a4-highgpu-onspot-slurm",
+        "gke-a4-onspot"
+    ), 2*60),
     ((
         "ml-a3-ultragpu-onspot-slurm", 
         "ml-a3-ultragpu-onspot-jbvms", 
@@ -72,6 +75,10 @@
         "ml-a3-highgpu-slurm", 
         "gke-a3-highgpu"
     ), 1*60),
+    ((
+        "ml-a3-highgpu-onspot-slurm", 
+        "gke-a3-highgpu-onspot"
+    ), 1*60),
 ]
 # TODO:
 # * Consider defining constraints (e.g. reservations used) as a tags within tests yamls