feat: add integration tests for htc example

build/int.cloudbuild.htc.yaml

@@ -0,0 +1,185 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 14400s
+steps:
+  - id: prepare
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      [
+        "/bin/bash",
+        "-c",
+        "source /usr/local/bin/task_helper_functions.sh && prepare_environment",
+      ]
+    env:
+      - "TF_VAR_org_id=$_ORG_ID"
+      - "TF_VAR_folder_id=$_FOLDER_ID"
+      - "TF_VAR_billing_account=$_BILLING_ACCOUNT"
+      - "TF_VAR_create_cloud_nat=true" # config-sync running on cluster VPC needs to access gitlab
+
+  - id: private-workerpool-init
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestPrivateWorkerPool --stage init --verbose"]
+
+  - id: private-workerpool-apply
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestPrivateWorkerPool --stage apply --verbose"]
+
+  - id: logging-bucket-init
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestLoggingBucket --stage init --verbose"]
+
+  - id: logging-bucket-apply
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestLoggingBucket --stage apply --verbose"]
+
+  - id: gitlab-init
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestGitLab --stage init --verbose"]
+
+  - id: gitlab-apply
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestGitLab --stage apply --verbose"]
+
+  - id: harness-init
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestSingleProjectHarness --stage init --verbose"]
+
+  - id: harness-apply
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestSingleProjectHarness --stage apply --verbose"]
+
+  - id: wait-startup-script
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestValidateStartupScript --stage verify --verbose"]
+
+  - id: bootstrap-gitlab-vm
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |-
+       gcloud storage cp gs://$(terraform -chdir=/workspace/test/setup/harness/gitlab output -raw gitlab_ssl_bucket_name)/gitlab.crt /usr/local/share/ca-certificates
+        update-ca-certificates
+        if (ls /etc/ssl/certs | grep gitlab.pem); then
+            cft test run TestBootstrapGitlabVM --stage verify --verbose || exit 1
+        else
+            echo ERROR: Was not able to download certificate and update system bundle. && exit 1
+        fi
+
+  - id: vpc-sc
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      [
+        "/bin/bash",
+        "-c",
+        "cft test run TestVPCSC --stage init --verbose && cft test run TestVPCSC --stage apply --verbose",
+      ]
+    env:
+      - "TF_VAR_org_id=$_ORG_ID"
+
+  - id: single-project-htc-init
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      [
+        "/bin/bash",
+        "-c",
+        "cft test run TestStandaloneHTCExample --stage init --verbose",
+      ]
+
+  - id: single-project-htc-apply
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      [
+        "/bin/bash",
+        "-c",
+        "cft test run TestStandaloneHTCExample --stage apply --verbose",
+      ]
+
+  - id: single-project-htc-verify
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      [
+        "/bin/bash",
+        "-c",
+        "cft test run TestStandaloneHTCExample --stage verify --verbose",
+      ]
+
+  - id: single-project-htc-teardown
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      [
+        "/bin/bash",
+        "-c",
+        "cft test run TestStandaloneHTCExample --stage teardown --verbose",
+      ]
+
+  - id: vpcsc-teardown
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      [
+        "/bin/bash",
+        "-c",
+        "cft test run TestVPCSC  --stage teardown --verbose",
+      ]
+
+  - id: harness-teardown
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      [
+        "/bin/bash",
+        "-c",
+        "cft test run TestSingleProjectHarness  --stage teardown --verbose",
+      ]
+
+  - id: gitlab-teardown
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestGitLab --stage teardown --verbose"]
+
+  - id: private-workerpool-teardown
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestPrivateWorkerPool --stage teardown --verbose"]
+
+  - id: logging-bucket-teardown
+    name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+    args:
+      ["/bin/bash", "-c", "cft test run TestLoggingBucket --stage teardown --verbose"]
+
+tags:
+  - "ci"
+  - "integration"
+substitutions:
+  _DOCKER_IMAGE_DEVELOPER_TOOLS: "cft/developer-tools"
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: "1.25"
+options:
+  machineType: E2_HIGHCPU_8
+  env:
+    - 'TF_PLUGIN_CACHE_DIR=""'
+    - 'TF_VAR_branch_name=$BRANCH_NAME'
+    - "TF_VAR_service_perimeter_mode=ENFORCE"
+    - "TF_VAR_deletion_protection=false"
+    - "TF_VAR_cloud_build_sa=$SERVICE_ACCOUNT_EMAIL"
+    - "TF_VAR_single_project=true"
+    - "TF_VAR_hpc=false"

examples/high-throughput-compute-standalone/5-appinfra.tf

@@ -16,26 +16,11 @@ locals {
   workload_args          = ["serve", "--logJSON"]
   workload_grpc_endpoint = "http://localhost:2002/main.LoadTestService/RunLibrary"
   workload_init_args = [
-    ["writedata", "--logJSON", "--showProgress=false", "--size", "104857600", "--parallel", "2", "--count", "10", "/data/read_dir/large"],
-    ["writedata", "--logJSON", "--showProgress=false", "--size", "10485760", "--parallel", "20", "--count", "100", "/data/read_dir/medium"],
-    ["writedata", "--logJSON", "--showProgress=false", "--size", "1048576", "--parallel", "20", "--count", "1000", "/data/read_dir/small"],
     ["writedata", "--logJSON", "--showProgress=false", "--size", "1048576", "--parallel", "20", "--count", "5", "/data/read_dir/small-low"],
-    ["gentasks", "--logJSON", "--count=10000", "--initMinMicros=1000000", "--minMicros=1000000", "--initReadDir=/data/read_dir/small", "/data/tasks/10k_1s_1s_read_small.jsonl.gz"],
     ["gentasks", "--logJSON", "--count=1000", "--initMinMicros=1000000", "--minMicros=1000000", "--initReadDir=/data/read_dir/small", "/data/tasks/1k_1s_1s_read_small.jsonl.gz"],
-    ["gentasks", "--logJSON", "--count=5000000", "--initMinMicros=30000000", "--minMicros=240000000", "--minMicros=360000000", "--initReadDir=/data/read_dir/small-low", "/data/tasks/5m_30s_360s_read_5_small.jsonl.gz"],
   ]
   test_configs = [
     { name = "hpa_1k_1s_s_read_small", testfile = "/data/tasks/1k_1s_1s_read_small.jsonl.gz", parallel = 0, description = "Autoscaling workers, 1 second compute and small folder read on initialization, 1,000 tasks for 1 second task compute." },
-    { name = "10_1k_1s_s_read_small", testfile = "/data/tasks/1k_1s_1s_read_small.jsonl.gz", parallel = 10, description = "10 workers, 1 second compute and small folder read on initialization, 1,000 tasks for 1 second compute." },
-    { name = "50_1k_1s_s_read_small", testfile = "/data/tasks/1k_1s_1s_read_small.jsonl.gz", parallel = 50, description = "50 workers, 1 second compute and small folder read on initialization, 1,000 tasks for 1 second compute." },
-    { name = "100_1k_1s_s_read_small", testfile = "/data/tasks/1k_1s_1s_read_small.jsonl.gz", parallel = 100, description = "100 workers, 1 second compute and small folder read on initialization, 1,000 tasks for 1 second compute." },
-    { name = "500_1k_1s_s_read_small", testfile = "/data/tasks/1k_1s_1s_read_small.jsonl.gz", parallel = 500, description = "500 workers, 1 second compute and small folder read on initialization, 1,000 tasks for 1 second compute." },
-    { name = "hpa_10k_1s_s_read_small", testfile = "/data/tasks/10k_1s_1s_read_small.jsonl.gz", parallel = 0, description = "Autoscaling workers, 1 second compute and small folder read on initialization, 10,000 tasks for 1 second compute." },
-    { name = "10_10k_1s_s_read_small", testfile = "/data/tasks/10k_1s_1s_read_small.jsonl.gz", parallel = 10, description = "10 workers, 1 second compute and small folder read on initialization, 10,000 tasks for 1 second compute." },
-    { name = "100_10k_1s_s_read_small", testfile = "/data/tasks/10k_1s_1s_read_small.jsonl.gz", parallel = 100, description = "100 workers, 1 second compute and small folder read on initialization, 10,000 tasks for 1 second compute." },
-    { name = "500_10k_1s_s_read_small", testfile = "/data/tasks/10k_1s_1s_read_small.jsonl.gz", parallel = 500, description = "500 workers, 1 second compute and small folder read on initialization, 10,000 tasks for 1 second compute." },
-    { name = "30000_5m_10s_s_read_small", testfile = "/data/tasks/5m_1s_3s_read_5_small.jsonl.gz", parallel = 30000, description = "30000 workers, 10 second compute and small folder read on initialization, 5 million tasks for 20-35 seconds of compute." },
-    { name = "hpa_5m_360s_s_read_small", testfile = "/data/tasks/5m_30s_360s_read_5_small.jsonl.gz", parallel = 0, description = "Autoscaling workers, 10 second compute and small folder read on initialization, 5 million tasks for 20-35 second compute." },
   ]
   test_configs_dict = {
     for config in local.test_configs :

examples/high-throughput-compute-standalone/README.md

@@ -126,18 +126,18 @@ links into the Console.
 | hsn\_bucket | Enable hierarchical namespace GCS buckets | `bool` | `false` | no |
 | lustre\_filesystem | The name of the Lustre filesystem | `string` | `"lustre-fs"` | no |
 | lustre\_gke\_support\_enabled | Enable GKE support for Lustre instance | `bool` | `true` | no |
-| max\_nodes\_ondemand | Maximum number of on-demand nodes | `number` | `32` | no |
-| max\_nodes\_spot | Maximum number of spot nodes | `number` | `3000` | no |
+| max\_nodes\_ondemand | Maximum number of on-demand nodes | `number` | `1` | no |
+| max\_nodes\_spot | Maximum number of spot nodes | `number` | `1` | no |
 | min\_nodes\_ondemand | Minimum number of on-demand nodes | `number` | `0` | no |
-| min\_nodes\_spot | Minimum number of spot nodes | `number` | `1` | no |
-| node\_machine\_type\_ondemand | Machine type for on-demand node pools | `string` | `"n2-standard-16"` | no |
-| node\_machine\_type\_spot | Machine type for spot node pools | `string` | `"n2-standard-64"` | no |
+| min\_nodes\_spot | Minimum number of spot nodes | `number` | `0` | no |
+| node\_machine\_type\_ondemand | Machine type for on-demand node pools | `string` | `"e2-standard-2"` | no |
+| node\_machine\_type\_spot | Machine type for spot node pools | `string` | `"e2-standard-2"` | no |
 | project\_id | The GCP project ID where resources will be created. | `string` | `"YOUR_PROJECT_ID"` | no |
 | pubsub\_exactly\_once | Enable Pub/Sub exactly once subscriptions | `bool` | `true` | no |
 | quota\_contact\_email | Contact email for quota requests | `string` | `""` | no |
 | regions | List of regions where GKE clusters should be created. Used for multi-region deployments. | `list(string)` | <pre>[<br>  "us-central1"<br>]</pre> | no |
 | scripts\_output | Output directory for testing scripts | `string` | `"./generated"` | no |
-| service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"ENFORCE"` | no |
+| service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"DRY_RUN"` | no |
 | service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no |
 | storage\_capacity\_gib | Capacity in GiB for the selected storage system (Parallelstore or Lustre) | `number` | `null` | no |
 | storage\_ip\_range | IP range for Storage peering, in CIDR notation | `string` | `"172.16.0.0/16"` | no |

examples/high-throughput-compute-standalone/agent/modules/config_apply/k8s/hpa.yaml.templ

@@ -18,8 +18,8 @@ metadata:
   name: ${name}
 spec:
   minReplicas: 1
-  maxReplicas: 50000
-  maxReplicas: 50000
+  maxReplicas: 2
+  maxReplicas: 2
   metrics:
   - external:
       metric:
@@ -87,13 +87,13 @@ spec:
       - name: agent
         resources:
           requests:
-            memory: "3Gi"
+            memory: "1Gi"
             cpu: "900m"
           limits:
-            cpu: "3900m"
-            memory: "7Gi"
-            cpu: "3900m"
-            memory: "7Gi"
+            cpu: "1000m"
+            memory: "3Gi"
+            cpu: "1000m"
+            memory: "3Gi"
         image: ${agent_image}
         args: ["serve", "pubsub-pull",
                # subscription to pull requests (tasks) from, topic to publish responses

examples/high-throughput-compute-standalone/agent/src/go.mod

@@ -1,26 +1,26 @@
 module github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent
 
 go 1.24
-toolchain go1.24.5
+toolchain go1.24.1
 
 require (
-	cloud.google.com/go/compute/metadata v0.7.0
-	cloud.google.com/go/pubsub v1.49.0
-	cloud.google.com/go/storage v1.55.0
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0
+	cloud.google.com/go/compute/metadata v0.5.2
+	cloud.google.com/go/pubsub v1.44.0
+	cloud.google.com/go/storage v1.44.0
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.2
 	github.com/bufbuild/protocompile v0.14.1
 	github.com/jhump/protoreflect v1.17.0
-	github.com/spf13/cobra v1.9.1
+	github.com/spf13/cobra v1.8.1
 	go.opencensus.io v0.24.0
-	go.opentelemetry.io/contrib/detectors/gcp v1.37.0
-	go.opentelemetry.io/otel v1.37.0
-	go.opentelemetry.io/otel/bridge/opencensus v1.37.0
-	go.opentelemetry.io/otel/sdk v1.37.0
-	go.opentelemetry.io/otel/sdk/metric v1.37.0
-	golang.org/x/oauth2 v0.30.0
-	google.golang.org/api v0.241.0
-	google.golang.org/grpc v1.73.0
-	google.golang.org/protobuf v1.36.6
+	go.opentelemetry.io/contrib/detectors/gcp v1.30.0
+	go.opentelemetry.io/otel v1.31.0
+	go.opentelemetry.io/otel/bridge/opencensus v1.31.0
+	go.opentelemetry.io/otel/sdk v1.31.0
+	go.opentelemetry.io/otel/sdk/metric v1.31.0
+	golang.org/x/oauth2 v0.23.0
+	google.golang.org/api v0.203.0
+	google.golang.org/grpc v1.67.1
+	google.golang.org/protobuf v1.35.1
 )
 
 require (

examples/high-throughput-compute-standalone/modules/gke-standard/README.md

@@ -129,8 +129,8 @@ limitations under the License.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | cluster\_index | Index of this cluster within its region (0-3) | `number` | n/a | yes |
-| cluster\_max\_cpus | Max CPU in cluster autoscaling resource limits | `number` | `10000` | no |
-| cluster\_max\_memory | Max memory in cluster autoscaling resource limits | `number` | `80000` | no |
+| cluster\_max\_cpus | Max CPU in cluster autoscaling resource limits | `number` | `100` | no |
+| cluster\_max\_memory | Max memory in cluster autoscaling resource limits | `number` | `1024` | no |
 | cluster\_name | Name of GKE cluster | `string` | `"gke-risk-research"` | no |
 | cluster\_service\_account | The service account for the GKE cluster | <pre>object({<br>    email = string<br>    id    = string<br>  })</pre> | n/a | yes |
 | create\_ondemand\_nodepool | Whether to create the on-demand node pool | `bool` | `true` | no |
@@ -153,14 +153,14 @@ limitations under the License.
 | maintenance\_end\_time | The end time for the maintenance window in RFC3339 format (e.g., '2024-09-18T04:00:00Z') | `string` | `"2024-09-18T04:00:00Z"` | no |
 | maintenance\_recurrence | The recurrence of the maintenance window in RRULE format (e.g., 'FREQ=WEEKLY;BYDAY=SA,SU') | `string` | `"FREQ=WEEKLY;BYDAY=SA,SU"` | no |
 | maintenance\_start\_time | The start time for the maintenance window in RFC3339 format (e.g., '2024-09-17T04:00:00Z') | `string` | `"2024-09-17T04:00:00Z"` | no |
-| max\_nodes\_ondemand | Maximum number of on-demand nodes in the node pool | `number` | `32` | no |
-| max\_nodes\_spot | Maximum number of spot nodes in the node pool | `number` | `3000` | no |
+| max\_nodes\_ondemand | Maximum number of on-demand nodes in the node pool | `number` | `1` | no |
+| max\_nodes\_spot | Maximum number of spot nodes in the node pool | `number` | `1` | no |
 | min\_master\_version | The minimum version of the master. GKE will auto-update the master to new versions, so this does not guarantee the current master version. | `string` | `"1.32.3"` | no |
 | min\_nodes\_ondemand | Minimum number of on-demand nodes in the node pool | `number` | `0` | no |
-| min\_nodes\_spot | Minimum number of spot nodes in the node pool | `number` | `1` | no |
+| min\_nodes\_spot | Minimum number of spot nodes in the node pool | `number` | `0` | no |
 | network | The vpc the cluster should be deployed to | `string` | `"default"` | no |
-| node\_machine\_type\_ondemand | Machine type for on-demand node pools in GKE clusters | `string` | `"n2-standard-16"` | no |
-| node\_machine\_type\_spot | Machine type for spot node pools in GKE clusters | `string` | `"n2-standard-64"` | no |
+| node\_machine\_type\_ondemand | Machine type for on-demand node pools in GKE clusters | `string` | `"e2-standard-2"` | no |
+| node\_machine\_type\_spot | Machine type for spot node pools in GKE clusters | `string` | `"e2-standard-2"` | no |
 | project\_id | The GCP project where the resources will be created | `string` | n/a | yes |
 | region | The region to host the cluster in | `string` | `"us-central1"` | no |
 | release\_channel | GKE release channel for clusters (RAPID, REGULAR, STABLE) | `string` | `"RAPID"` | no |