From fa3114be1718ef6fb9c8c70c3e381b35bcfac675 Mon Sep 17 00:00:00 2001 From: Amanda Karina Lopes de Oliveira Date: Fri, 26 Dec 2025 12:30:12 -0300 Subject: [PATCH 1/7] fix: examples/hpc/6-appsource/fsi-montecarlo-on-batch.yaml (#529) --- examples/hpc/6-appsource/fsi-montecarlo-on-batch.yaml | 4 ++-- test/integration/hpc/hpc_e2e_test.go | 4 ++-- test/setup/main.tf | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/hpc/6-appsource/fsi-montecarlo-on-batch.yaml b/examples/hpc/6-appsource/fsi-montecarlo-on-batch.yaml index b23db8541..ea7f4334e 100644 --- a/examples/hpc/6-appsource/fsi-montecarlo-on-batch.yaml +++ b/examples/hpc/6-appsource/fsi-montecarlo-on-batch.yaml @@ -97,7 +97,7 @@ deployment_groups: ' - id: data_bucket - source: community/modules/file-system/cloud-storage-bucket + source: modules/file-system/cloud-storage-bucket settings: name_prefix: data_bucket random_suffix: true @@ -108,7 +108,7 @@ deployment_groups: source: community/modules/compute/notebook use: [data_bucket] settings: - machine_type: n1-standard-4 + machine_type: n2-standard-4 instance_image: {"family": "workbench-instances", "name": null, "project": "cloud-notebooks-managed"} - id: bq_subscription diff --git a/test/integration/hpc/hpc_e2e_test.go b/test/integration/hpc/hpc_e2e_test.go index c499119e7..9fe7223d2 100644 --- a/test/integration/hpc/hpc_e2e_test.go +++ b/test/integration/hpc/hpc_e2e_test.go @@ -232,7 +232,7 @@ func TestHPCMonteCarloE2E(t *testing.T) { runBatchJobs(t) t.Log("Validating Batch Jobs:") - for count := 0; count < 15; count++ { + for count := 0; count < 30; count++ { jobsOutput, err := k8s.RunKubectlAndGetOutputE(t, k8sOpts, "get", "jobs", "-n", "hpc-team-b-development", "-o", "json") if err != nil { t.Fatal(err) @@ -300,7 +300,7 @@ func TestHPCMonteCarloE2E(t *testing.T) { t.Fatal(err) } // validate job finished - for count := 0; count < 15; count++ { + for count := 0; count < 30; count++ { jobsOutput, err := k8s.RunKubectlAndGetOutputE(t, k8sOpts, "-n", "hpc-team-a-development", "logs", "jobs/mnist-training-job", "-c", "tensorflow") if err != nil { t.Log(err) diff --git a/test/setup/main.tf b/test/setup/main.tf index 2096e8055..88918e613 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -83,6 +83,7 @@ module "seed_project" { "multiclusterservicediscovery.googleapis.com", "networkmanagement.googleapis.com", "networkservices.googleapis.com", + "notebooks.googleapis.com", "orgpolicy.googleapis.com", "secretmanager.googleapis.com", "servicedirectory.googleapis.com", From 75b401cbf668033bc67e92bd9483df696f3aa913 Mon Sep 17 00:00:00 2001 From: marcelorobj Date: Mon, 1 Dec 2025 10:22:59 -0300 Subject: [PATCH 2/7] Feat: add default infra to htc (#5) --- 5-appinfra/modules/htc-infra/README.md | 147 ++ 5-appinfra/modules/htc-infra/agent/README.md | 179 +++ .../htc-infra/agent/docs/bigquery_pattern.png | Bin 0 -> 37560 bytes .../htc-infra/agent/docs/file_pattern.png | Bin 0 -> 36128 bytes .../htc-infra/agent/docs/pubsub_pattern.png | Bin 0 -> 45806 bytes .../agent/docs/pubsub_push_pattern.png | Bin 0 -> 45041 bytes .../k8s/adapter_new_resource_model.yaml.templ | 209 +++ .../modules/config_apply/k8s/agent_job.templ | 107 ++ .../config_apply/k8s/controller_job.templ | 64 + .../modules/config_apply/k8s/hpa.yaml.templ | 111 ++ .../agent/modules/config_apply/k8s/job.templ | 66 + .../k8s/parallelstore/agent_job.templ | 104 ++ .../k8s/parallelstore/controller_job.templ | 53 + .../k8s/parallelstore/hpa.yaml.templ | 118 ++ .../config_apply/k8s/parallelstore/job.templ | 55 + .../k8s/parallelstore/volume.yaml.templ | 37 + .../k8s/parallelstore/volume_claim.yaml.templ | 25 + .../config_apply/k8s/test_config.sh.templ | 34 + .../config_apply/k8s/volume.yaml.templ | 32 + .../config_apply/k8s/volume_claim.yaml.templ | 27 + .../agent/modules/config_apply/main.tf | 305 ++++ .../agent/modules/config_apply/outputs.tf | 30 + .../agent/modules/config_apply/variables.tf | 190 +++ .../agent/modules/config_apply/versions.tf | 39 + .../htc-infra/agent/modules/gke/.gitignore | 1 + .../dashboards/risk-platform-overview.json | 1255 +++++++++++++++++ .../htc-infra/agent/modules/gke/main.tf | 125 ++ .../htc-infra/agent/modules/gke/outputs.tf | 53 + .../htc-infra/agent/modules/gke/variables.tf | 176 +++ .../htc-infra/agent/modules/gke/versions.tf | 35 + .../modules/htc-infra/agent/src/.gitignore | 2 + .../modules/htc-infra/agent/src/Dockerfile | 34 + .../modules/htc-infra/agent/src/gcp/google.go | 207 +++ .../htc-infra/agent/src/gcp/storage.go | 155 ++ 5-appinfra/modules/htc-infra/agent/src/go.mod | 67 + 5-appinfra/modules/htc-infra/agent/src/go.sum | 226 +++ .../modules/htc-infra/agent/src/main.go | 128 ++ .../htc-infra/agent/src/protoio/backend.go | 296 ++++ .../htc-infra/agent/src/protoio/convert.go | 116 ++ .../modules/htc-infra/agent/src/protoio/io.go | 144 ++ .../htc-infra/agent/src/protoio/io_test.go | 122 ++ .../agent/src/protoio/test_service.pb.go | 271 ++++ .../agent/src/protoio/test_service.proto | 51 + .../agent/src/protoio/test_service_grpc.pb.go | 137 ++ .../htc-infra/agent/src/serve/bigquery_rdf.go | 149 ++ .../modules/htc-infra/agent/src/serve/file.go | 69 + .../htc-infra/agent/src/serve/pubsub.go | 168 +++ .../htc-infra/agent/src/serve/pubsub_push.go | 138 ++ .../htc-infra/agent/src/serve/serve.go | 47 + .../htc-infra/agent/src/stats/stats.go | 192 +++ .../htc-infra/agent/src/stats/stats_accum.go | 186 +++ .../htc-infra/agent/src/stats/throttle.go | 181 +++ .../htc-infra/agent/src/stats/timings.go | 85 ++ .../htc-infra/agent/src/stats/timings_test.go | 53 + .../htc-infra/agent/src/test/generate.go | 45 + .../modules/htc-infra/agent/src/test/grpc.go | 60 + .../htc-infra/agent/src/test/pubsub.go | 233 +++ .../htc-infra/agent/src/test/source.go | 116 ++ .../htc-infra/american-option/.gitignore | 6 + .../htc-infra/american-option/Dockerfile | 28 + .../htc-infra/american-option/README.md | 98 ++ .../american-option/american_option.py | 149 ++ .../modules/htc-infra/american-option/main.py | 137 ++ .../htc-infra/american-option/request.proto | 149 ++ .../htc-infra/american-option/requirements.in | 19 + .../american-option/requirements.txt | 162 +++ .../htc-infra/american-option/response.proto | 22 + .../htc-infra/american-option/service.proto | 26 + .../htc-infra/american-option/setup.sh | 45 + .../htc-infra/american-option/test_data.py | 177 +++ 5-appinfra/modules/htc-infra/apis.tf | 58 + .../compute-classes/gpu-spot-capacity.yaml | 42 + .../compute-classes/h3-spot-capacity.yaml | 30 + .../compute-classes/kustomization.yaml | 20 + .../compute-classes/n-spot-capacity.yaml | 45 + .../htc-infra/kubernetes/kueue/README.md | 68 + .../kueue/base/apiserver-reader.yaml | 27 + .../kueue/base/kueue-manager-config.yaml | 62 + .../kubernetes/kueue/base/kustomization.yaml | 39 + .../kubernetes/kueue/base/promethus-gmp.yaml | 84 ++ .../kueue/dashboard/kueue-dashboard.json | 700 +++++++++ .../overlays/gke-autopilot/kustomization.yaml | 27 + .../overlays/gke-standard/kustomization.yaml | 27 + .../kubernetes/placeholder-pods.yaml | 50 + .../kubernetes/priority-classes/classes.yaml | 40 + .../priority-classes/kustomization.yaml | 18 + .../storage/fio-testing/.dockerignore | 3 + .../kubernetes/storage/fio-testing/Dockerfile | 32 + .../kubernetes/storage/fio-testing/README.md | 152 ++ .../storage/fio-testing/entrypoint.sh | 122 ++ .../k8s/gcs/delete-data/cleanup-disk.yaml | 57 + .../k8s/gcs/delete-data/configmap.yaml | 116 ++ .../k8s/gcs/delete-data/kustomization.yaml | 202 +++ .../fio-testing/k8s/gcs/fio-config.yaml | 44 + .../k8s/gcs/fio-test/fio-config.yaml | 44 + .../k8s/gcs/fio-test/gcs-testing.yaml | 81 ++ .../k8s/gcs/fio-test/kustomization.yaml | 101 ++ .../fio-testing/k8s/gcs/gcs-testing.yaml | 74 + .../gcs/generate-data/generate-data-gcs.yaml | 174 +++ .../k8s/gcs/generate-data/kustomization.yaml | 100 ++ .../fio-testing/k8s/gcs/kustomization.yaml | 62 + .../storage/fio-testing/k8s/helper.yaml | 42 + .../k8s/lustre/base/kustomization.yaml | 20 + .../k8s/lustre/base/lustre-pv.yaml | 32 + .../k8s/lustre/base/lustre-pvc.yaml | 26 + .../overlays/delete-data/cleanup-disk.yaml | 76 + .../overlays/delete-data/kustomization.yaml | 195 +++ .../overlays/generate-data/configmap.yaml | 113 ++ .../overlays/generate-data/generate-data.yaml | 65 + .../overlays/generate-data/kustomization.yaml | 202 +++ .../overlays/large-files-test/fio-config.yaml | 44 + .../large-files-test/kustomization.yaml | 215 +++ .../large-files-test/lustre-testing.yaml | 69 + .../overlays/small-files-test/fio-config.yaml | 45 + .../small-files-test/kustomization.yaml | 215 +++ .../small-files-test/lustre-testing.yaml | 69 + .../k8s/parallelstore/base/kustomization.yaml | 20 + .../parallelstore/base/parallelstore-pv.yaml | 33 + .../parallelstore/base/parallelstore-pvc.yaml | 25 + .../k8s/parallelstore/fio-config.yaml | 44 + .../k8s/parallelstore/kustomization.yaml | 208 +++ .../overlays/delete-data/cleanup-disk.yaml | 74 + .../overlays/delete-data/configmap.yaml | 116 ++ .../overlays/delete-data/kustomization.yaml | 202 +++ .../overlays/fio-test/fio-config.yaml | 67 + .../overlays/fio-test/kustomization.yaml | 234 +++ .../fio-test/parallelstore-testing.yaml | 77 + .../overlays/generate-data/configmap.yaml | 113 ++ .../overlays/generate-data/generate-data.yaml | 65 + .../overlays/generate-data/kustomization.yaml | 242 ++++ .../large-files-test-node/fio-config.yaml | 44 + .../large-files-test-node/kustomization.yaml | 234 +++ .../parallelstore-testing.yaml | 69 + .../large-files-test-sidecar/fio-config.yaml | 45 + .../kustomization.yaml | 234 +++ .../parallelstore-testing.yaml | 77 + .../small-files-test-node/fio-config.yaml | 45 + .../small-files-test-node/kustomization.yaml | 235 +++ .../parallelstore-testing.yaml | 69 + .../small-files-test-sidecar/fio-config.yaml | 45 + .../kustomization.yaml | 234 +++ .../parallelstore-testing.yaml | 77 + .../k8s/parallelstore/parallelstore-pv.yaml | 33 + .../k8s/parallelstore/parallelstore-pvc.yaml | 25 + .../parallelstore/parallelstore-testing.yaml | 60 + .../storage/gcsfuse-nodemount/README.md | 111 ++ .../storage/gcsfuse-nodemount/cloudbuild.yaml | 26 + .../gcsfuse-nodemount/k8s/daemonset.yaml | 137 ++ .../gcsfuse-nodemount/k8s/example-pod.yaml | 38 + .../gcsfuse-nodemount/k8s/kustomization.yaml | 23 + .../gcsfuse-nodemount/k8s/namespace.yaml | 18 + .../parallelstore-transfer-tool/.Dockerignore | 3 + .../parallelstore-transfer-tool/Dockerfile | 31 + .../parallelstore-transfer-tool/README.md | 111 ++ .../k8s/kustomization.yaml | 48 + .../k8s/sample-job.yaml | 37 + .../k8s/serviceaccount.yaml | 18 + .../parallelstore-transfer-tool/main.py | 183 +++ .../requirements.in | 1 + .../requirements.txt | 51 + .../storage/parallelstore/README.md | 124 ++ .../storage/parallelstore/cloudbuild.yaml | 18 + .../parallelstore/example-workload.yaml | 57 + .../storage/parallelstore/image/Dockerfile | 159 +++ .../parallelstore/image/generate-config.sh | 26 + .../image/supervisord-template.conf | 9 + .../k8s/daemonset-access-points.yaml | 26 + .../storage/parallelstore/k8s/daemonset.yaml | 107 ++ .../parallelstore/k8s/kustomization.yaml | 22 + .../modules/htc-infra/loadtest/.gitignore | 5 + .../modules/htc-infra/loadtest/Dockerfile | 40 + .../modules/htc-infra/loadtest/README.md | 99 ++ .../modules/htc-infra/loadtest/generate.go | 221 +++ 5-appinfra/modules/htc-infra/loadtest/go.mod | 64 + 5-appinfra/modules/htc-infra/loadtest/go.sum | 232 +++ 5-appinfra/modules/htc-infra/loadtest/main.go | 204 +++ .../modules/htc-infra/loadtest/request.proto | 60 + .../modules/htc-infra/loadtest/response.proto | 42 + .../modules/htc-infra/loadtest/service.proto | 28 + .../modules/htc-infra/loadtest/simulator.go | 293 ++++ .../modules/htc-infra/loadtest/utils.go | 516 +++++++ 5-appinfra/modules/htc-infra/main.tf | 358 +++++ .../modules/artifact-registry/README.md | 83 ++ .../modules/artifact-registry/main.tf | 46 + .../modules/artifact-registry/outputs.tf | 33 + .../modules/artifact-registry/variables.tf | 42 + .../modules/artifact-registry/versions.tf | 39 + .../htc-infra/modules/builder/README.md | 106 ++ .../modules/htc-infra/modules/builder/main.tf | 117 ++ .../htc-infra/modules/builder/outputs.tf | 28 + .../htc-infra/modules/builder/variables.tf | 53 + .../htc-infra/modules/builder/versions.tf | 39 + .../htc-infra/modules/kubectl/README.md | 60 + .../htc-infra/modules/kubectl/versions.tf | 31 + .../htc-infra/modules/lustre/README.md | 100 ++ .../modules/htc-infra/modules/lustre/main.tf | 54 + .../htc-infra/modules/lustre/outputs.tf | 53 + .../htc-infra/modules/lustre/variables.tf | 64 + .../htc-infra/modules/lustre/versions.tf | 39 + .../htc-infra/modules/parallelstore/README.md | 98 ++ .../htc-infra/modules/parallelstore/main.tf | 52 + .../modules/parallelstore/outputs.tf | 63 + .../modules/parallelstore/variables.tf | 58 + .../modules/parallelstore/versions.tf | 39 + .../htc-infra/modules/project/README.md | 98 ++ .../modules/htc-infra/modules/project/main.tf | 208 +++ .../htc-infra/modules/project/outputs.tf | 18 + .../htc-infra/modules/project/variables.tf | 29 + .../htc-infra/modules/project/versions.tf | 31 + .../modules/pubsub-subscriptions/README.md | 85 ++ .../modules/pubsub-subscriptions/main.tf | 98 ++ .../modules/pubsub-subscriptions/outputs.tf | 13 + .../pubsub_json_schema.txt | 7 + .../modules/pubsub-subscriptions/variables.tf | 49 + .../modules/pubsub-subscriptions/versions.tf | 43 + .../modules/htc-infra/modules/quota/README.md | 116 ++ .../modules/htc-infra/modules/quota/main.tf | 51 + .../htc-infra/modules/quota/outputs.tf | 23 + .../htc-infra/modules/quota/variables.tf | 59 + .../htc-infra/modules/quota/versions.tf | 31 + .../modules/region-analysis/README.md | 99 ++ .../htc-infra/modules/region-analysis/main.tf | 58 + .../modules/region-analysis/output.tf | 39 + .../modules/region-analysis/variables.tf | 19 + .../modules/region-analysis/versions.tf | 31 + 5-appinfra/modules/htc-infra/outputs.tf | 23 + 5-appinfra/modules/htc-infra/sql/README.md | 21 + .../modules/htc-infra/sql/agent_stats.sql | 33 + .../sql/agent_summary_by_instance.sql | 59 + .../modules/htc-infra/sql/log_stats.sql | 39 + .../htc-infra/sql/pubsub_messages_joined.sql | 100 ++ .../htc-infra/sql/pubsub_messages_summary.sql | 73 + 5-appinfra/modules/htc-infra/variables.tf | 248 ++++ 5-appinfra/modules/htc-infra/versions.tf | 43 + 5-appinfra/modules/htc-infra/vpc-sc.tf | 122 ++ examples/htc/4-appfactory/terraform.tfvars | 45 + .../5-appinfra/htc/cloudbuild-tf-apply.yaml | 81 ++ .../5-appinfra/htc/cloudbuild-tf-plan.yaml | 54 + .../5-appinfra/htc/envs/production/README.md | 133 ++ .../5-appinfra/htc/envs/production/backend.tf | 22 + .../5-appinfra/htc/envs/production/main.tf | 66 + .../5-appinfra/htc/envs/production/outputs.tf | 18 + .../5-appinfra/htc/envs/production/remote.tf | 74 + .../htc/envs/production/variables.tf | 134 ++ .../htc/envs/production/versions.tf | 43 + examples/htc/5-appinfra/htc/modules | 1 + examples/htc/5-appinfra/htc/tf-wrapper.sh | 343 +++++ examples/htc/6-appsource/cloudbuild.yaml | 119 ++ examples/htc/README.md | 264 ++++ 249 files changed, 23654 insertions(+) create mode 100644 5-appinfra/modules/htc-infra/README.md create mode 100644 5-appinfra/modules/htc-infra/agent/README.md create mode 100644 5-appinfra/modules/htc-infra/agent/docs/bigquery_pattern.png create mode 100644 5-appinfra/modules/htc-infra/agent/docs/file_pattern.png create mode 100644 5-appinfra/modules/htc-infra/agent/docs/pubsub_pattern.png create mode 100644 5-appinfra/modules/htc-infra/agent/docs/pubsub_push_pattern.png create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/agent_job.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/controller_job.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/job.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/agent_job.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/controller_job.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/hpa.yaml.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/job.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/volume.yaml.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/volume_claim.yaml.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/test_config.sh.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/volume.yaml.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/volume_claim.yaml.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/variables.tf create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/versions.tf create mode 100644 5-appinfra/modules/htc-infra/agent/modules/gke/.gitignore create mode 100644 5-appinfra/modules/htc-infra/agent/modules/gke/dashboards/risk-platform-overview.json create mode 100644 5-appinfra/modules/htc-infra/agent/modules/gke/main.tf create mode 100644 5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf create mode 100644 5-appinfra/modules/htc-infra/agent/modules/gke/versions.tf create mode 100644 5-appinfra/modules/htc-infra/agent/src/.gitignore create mode 100644 5-appinfra/modules/htc-infra/agent/src/Dockerfile create mode 100644 5-appinfra/modules/htc-infra/agent/src/gcp/google.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/gcp/storage.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/go.mod create mode 100644 5-appinfra/modules/htc-infra/agent/src/go.sum create mode 100644 5-appinfra/modules/htc-infra/agent/src/main.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/protoio/backend.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/protoio/convert.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/protoio/io.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/protoio/test_service.pb.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/protoio/test_service.proto create mode 100644 5-appinfra/modules/htc-infra/agent/src/protoio/test_service_grpc.pb.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/serve/bigquery_rdf.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/serve/file.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/serve/pubsub.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/serve/pubsub_push.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/serve/serve.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/stats/stats.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/stats/stats_accum.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/stats/throttle.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/stats/timings.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/stats/timings_test.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/test/generate.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/test/grpc.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/test/pubsub.go create mode 100644 5-appinfra/modules/htc-infra/agent/src/test/source.go create mode 100644 5-appinfra/modules/htc-infra/american-option/.gitignore create mode 100644 5-appinfra/modules/htc-infra/american-option/Dockerfile create mode 100644 5-appinfra/modules/htc-infra/american-option/README.md create mode 100755 5-appinfra/modules/htc-infra/american-option/american_option.py create mode 100755 5-appinfra/modules/htc-infra/american-option/main.py create mode 100644 5-appinfra/modules/htc-infra/american-option/request.proto create mode 100644 5-appinfra/modules/htc-infra/american-option/requirements.in create mode 100644 5-appinfra/modules/htc-infra/american-option/requirements.txt create mode 100644 5-appinfra/modules/htc-infra/american-option/response.proto create mode 100644 5-appinfra/modules/htc-infra/american-option/service.proto create mode 100755 5-appinfra/modules/htc-infra/american-option/setup.sh create mode 100755 5-appinfra/modules/htc-infra/american-option/test_data.py create mode 100644 5-appinfra/modules/htc-infra/apis.tf create mode 100644 5-appinfra/modules/htc-infra/kubernetes/compute-classes/gpu-spot-capacity.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/compute-classes/h3-spot-capacity.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/compute-classes/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/compute-classes/n-spot-capacity.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/kueue/README.md create mode 100644 5-appinfra/modules/htc-infra/kubernetes/kueue/base/apiserver-reader.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/kueue/base/kueue-manager-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/kueue/base/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/kueue/base/promethus-gmp.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/kueue/dashboard/kueue-dashboard.json create mode 100644 5-appinfra/modules/htc-infra/kubernetes/kueue/overlays/gke-autopilot/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/kueue/overlays/gke-standard/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/placeholder-pods.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/priority-classes/classes.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/priority-classes/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/.dockerignore create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/Dockerfile create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/README.md create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/entrypoint.sh create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/cleanup-disk.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/configmap.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/gcs-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/gcs-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/generate-data/generate-data-gcs.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/generate-data/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/helper.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/lustre-pv.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/lustre-pvc.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/delete-data/cleanup-disk.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/delete-data/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/configmap.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/generate-data.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/lustre-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/lustre-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/parallelstore-pv.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/parallelstore-pvc.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/cleanup-disk.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/configmap.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/parallelstore-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/configmap.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/generate-data.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/parallelstore-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/parallelstore-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/parallelstore-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/fio-config.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/parallelstore-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-pv.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-pvc.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-testing.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/README.md create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/cloudbuild.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/daemonset.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/example-pod.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/namespace.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/.Dockerignore create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/Dockerfile create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/README.md create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/sample-job.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/serviceaccount.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/main.py create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/requirements.in create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/requirements.txt create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/README.md create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/cloudbuild.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/example-workload.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/Dockerfile create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/generate-config.sh create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/supervisord-template.conf create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/daemonset-access-points.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/daemonset.yaml create mode 100644 5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/kustomization.yaml create mode 100644 5-appinfra/modules/htc-infra/loadtest/.gitignore create mode 100644 5-appinfra/modules/htc-infra/loadtest/Dockerfile create mode 100644 5-appinfra/modules/htc-infra/loadtest/README.md create mode 100644 5-appinfra/modules/htc-infra/loadtest/generate.go create mode 100644 5-appinfra/modules/htc-infra/loadtest/go.mod create mode 100644 5-appinfra/modules/htc-infra/loadtest/go.sum create mode 100644 5-appinfra/modules/htc-infra/loadtest/main.go create mode 100644 5-appinfra/modules/htc-infra/loadtest/request.proto create mode 100644 5-appinfra/modules/htc-infra/loadtest/response.proto create mode 100644 5-appinfra/modules/htc-infra/loadtest/service.proto create mode 100644 5-appinfra/modules/htc-infra/loadtest/simulator.go create mode 100644 5-appinfra/modules/htc-infra/loadtest/utils.go create mode 100644 5-appinfra/modules/htc-infra/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/artifact-registry/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/artifact-registry/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/artifact-registry/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/modules/artifact-registry/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/artifact-registry/versions.tf create mode 100644 5-appinfra/modules/htc-infra/modules/builder/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/builder/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/builder/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/modules/builder/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/builder/versions.tf create mode 100644 5-appinfra/modules/htc-infra/modules/kubectl/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/kubectl/versions.tf create mode 100644 5-appinfra/modules/htc-infra/modules/lustre/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/lustre/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/lustre/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/modules/lustre/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/lustre/versions.tf create mode 100644 5-appinfra/modules/htc-infra/modules/parallelstore/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/parallelstore/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/parallelstore/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/modules/parallelstore/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/parallelstore/versions.tf create mode 100644 5-appinfra/modules/htc-infra/modules/project/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/project/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/project/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/modules/project/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/project/versions.tf create mode 100644 5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/pubsub_json_schema.txt create mode 100644 5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/versions.tf create mode 100644 5-appinfra/modules/htc-infra/modules/quota/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/quota/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/quota/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/modules/quota/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/quota/versions.tf create mode 100644 5-appinfra/modules/htc-infra/modules/region-analysis/README.md create mode 100644 5-appinfra/modules/htc-infra/modules/region-analysis/main.tf create mode 100644 5-appinfra/modules/htc-infra/modules/region-analysis/output.tf create mode 100644 5-appinfra/modules/htc-infra/modules/region-analysis/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/region-analysis/versions.tf create mode 100644 5-appinfra/modules/htc-infra/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/sql/README.md create mode 100644 5-appinfra/modules/htc-infra/sql/agent_stats.sql create mode 100644 5-appinfra/modules/htc-infra/sql/agent_summary_by_instance.sql create mode 100644 5-appinfra/modules/htc-infra/sql/log_stats.sql create mode 100644 5-appinfra/modules/htc-infra/sql/pubsub_messages_joined.sql create mode 100644 5-appinfra/modules/htc-infra/sql/pubsub_messages_summary.sql create mode 100644 5-appinfra/modules/htc-infra/variables.tf create mode 100644 5-appinfra/modules/htc-infra/versions.tf create mode 100644 5-appinfra/modules/htc-infra/vpc-sc.tf create mode 100644 examples/htc/4-appfactory/terraform.tfvars create mode 100644 examples/htc/5-appinfra/htc/cloudbuild-tf-apply.yaml create mode 100644 examples/htc/5-appinfra/htc/cloudbuild-tf-plan.yaml create mode 100644 examples/htc/5-appinfra/htc/envs/production/README.md create mode 100644 examples/htc/5-appinfra/htc/envs/production/backend.tf create mode 100644 examples/htc/5-appinfra/htc/envs/production/main.tf create mode 100644 examples/htc/5-appinfra/htc/envs/production/outputs.tf create mode 100644 examples/htc/5-appinfra/htc/envs/production/remote.tf create mode 100644 examples/htc/5-appinfra/htc/envs/production/variables.tf create mode 100644 examples/htc/5-appinfra/htc/envs/production/versions.tf create mode 120000 examples/htc/5-appinfra/htc/modules create mode 100755 examples/htc/5-appinfra/htc/tf-wrapper.sh create mode 100644 examples/htc/6-appsource/cloudbuild.yaml create mode 100644 examples/htc/README.md diff --git a/5-appinfra/modules/htc-infra/README.md b/5-appinfra/modules/htc-infra/README.md new file mode 100644 index 000000000..7cbaaab67 --- /dev/null +++ b/5-appinfra/modules/htc-infra/README.md @@ -0,0 +1,147 @@ + +# Load Test for HTC + +## Overview + +This is a general purpose gRPC load test. + +See the details of running the program in its [src/README.md](src/README.md). See below for +deployment on Google Cloud. + +## Deployment with Cloud Shell + +The following link will walk you through a quick start in Cloud Shell: + +[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://shell.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fgooglecloudplatform%2Frisk-and-research-blueprints&cloudshell_git_branch=main&cloudshell_workspace=examples%2Frisk%2Floadtest&cloudshell_tutorial=QUICKSTART.md&show=terminal) + +## Deploy with Terraform + +### Requirements + +You must have have the following installed: +* `gcloud` installed (see [installation](https://cloud.google.com/sdk/docs/install)) +* `kubectl` installed (see [install tools](https://kubernetes.io/docs/tasks/tools/)) +* A bash-based shell (Linux or Mac OS/X) + +Note that Cloud Shell meets the requirements. + +### Configuration + +Create `terraform.tfvars` with the following content: +``` +project_id="" +region="" +zones=["", "", ""] +``` + +For example in us-central1: +``` +project_id="" +region="us-central1" +zones=["a", "b", "c", "f"] +``` + +For example in europe-west1: +``` +project_id="" +region="europe-west1" +zones=["b", "c", "d"] +``` + +### Create infrastructure + +Authorize `gcloud` if needed: +```sh +gcloud auth login --quiet --update-adc +``` + +Update the `gcloud` project: + +```bash +gcloud config set project YOUR_PROJECT_ID +``` + +You may need to enable some basic APIs for Terraform to work: +```sh +gcloud services enable iam.googleapis.com cloudresourcemanager.googleapis.com +``` + +Initialize and run terraform: +```sh +terraform init +terraform apply +``` + +NOTE: While running the terraform if the APIs are newly enabled, there may be +timing errors and terraform apply will need to be re-run. + +## Seeing infrastructure & Running Test Workloads + +### See what's from terraform + +Inspect the possible run scripts: +```sh +terraform output +``` + +Key variable outputs: + * local_test_scripts contain a list of shell scripts which you can run for different loadtests. + * get_credentials is the command line to fetch the credentials for kubectl. + * lookerstudio_create_dashboard_url is a link to create a new Lookerstudio Dashboard from a template. + * monitoring_dashboard_url is a custom made monitoring dashboard for loadtest. + +### Running the GUI + +Create a virtual environment: +```sh +python3 -m venv ui/.venv +ui/.venv/bin/python3 -m pip install -r ui/requirements.txt +``` + +Run the Gradio dashboard: +```sh +ui/.venv/bin/python3 ui/main.py generated/config.yaml +``` + +Use port 8080 or preview 8080 in the Cloud Shell (Webpreview). This allows you to load +tests, inspect the jobs from BigQuery (similar to the dashboard), and has some deep +links into the Console. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| access\_level\_name | (VPC-SC) Access Level full name. When providing this variable, additional identities will be added to the access level, these are required to work within an enforced VPC-SC Perimeter. | `string` | `null` | no | +| additional\_quota\_enabled | Enable quota requests for additional resources | `bool` | `false` | no | +| admin\_project | The admin project where cloudbuild/cloudrun configurations will be managed. | `string` | n/a | yes | +| cluster\_project\_id | The GCP project ID where the cluster is created. | `string` | n/a | yes | +| cluster\_project\_number | The GCP project ID where the cluster is created. | `string` | n/a | yes | +| enable\_csi\_parallelstore | Enable the Parallelstore CSI Driver | `bool` | `true` | no | +| env | The environment to prepare (ex. development) | `string` | n/a | yes | +| gke\_cluster\_names | GKE Cluster Name to be used in configurations | `list(string)` | n/a | yes | +| hsn\_bucket | Enable hierarchical namespace GCS buckets | `bool` | `false` | no | +| infra\_project | The infrastructure project where resources will be managed. | `string` | n/a | yes | +| network\_name | VPC Network Name | `string` | n/a | yes | +| network\_self\_link | VPC Network self link | `string` | n/a | yes | +| parallelstore\_deployment\_type | Parallelstore Instance deployment type (SCRATCH or PERSISTENT) | `string` | `"SCRATCH"` | no | +| pubsub\_exactly\_once | Enable Pub/Sub exactly once subscriptions | `bool` | `true` | no | +| quota\_contact\_email | Contact email for quota requests | `string` | `""` | no | +| region | The region where the cloud resources will be deployed. | `string` | n/a | yes | +| regions | List of regions where GKE clusters should be created. Used for multi-region deployments. | `list(string)` | 
[
  "us-central1"
]
| no | +| service\_name | service name (e.g. 'transactionhistory') | `string` | n/a | yes | +| service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"DRY_RUN"` | no | +| service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no | +| storage\_capacity\_gib | Capacity in GiB for the selected storage system (Parallelstore or Lustre) | `number` | `null` | no | +| storage\_locations | Map of region to location (zone) for storage instances e.g. {"us-central1" = "us-central1-a"} | `map(string)` | `{}` | no | +| storage\_type | The type of storage system to deploy (PARALLELSTORE, LUSTRE, or null for none) | `string` | `null` | no | +| team | Environment Team, must be the same as the fleet scope team | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| test\_scripts | Test configuration shell scripts | +| ui\_config | Yaml configuration for UI deployment | + + diff --git a/5-appinfra/modules/htc-infra/agent/README.md b/5-appinfra/modules/htc-infra/agent/README.md new file mode 100644 index 000000000..9646632e4 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/README.md @@ -0,0 +1,179 @@ + +# Agent for HTC + +## Overview + +The Agent for HTC is a tool for deploying [Unary GRPC](https://grpc.io/docs/what-is-grpc/core-concepts/#unary-rpc) services. This an RPC service +that takes in a [protobuf message](https://protobuf.dev/overview/) and returns a +protobuf message. + +This can use the [loadtest](../loadtest) or [american-option](../american-option) as +workloads and will be used in this example. Your own workload can be used as well. + +## Agent Modes + +The Agent is intended to be deployed as a sidecar to the gRPC service. It can +be deployed in [Cloud Run](https://cloud.google.com/run) or [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine). Cloud Run +provides scale-to-zero, rapid scaling, and a fully managed service. Google +Kubernetes Engine provides immense flexibility, scalability, and enterprise control. + +The following examples are focused in Cloud Run but some can run on Google Kubernetes +Engine (or locally). + +### BigQuery + +![BigQuery Pattern](docs/bigquery_pattern.png "BigQuery Pattern") + +The agent will be deployed alongside the workload in the same container on Cloud Run. It will +receive HTTP requests from [BigQuery Remote Functions](https://cloud.google.com/bigquery/docs/remote-functions), converting +JSON requests into protobuf, dispatching it to the gRPC service, and returning the +results back to BigQuery as JSON. + +This enables flexible data anslysis from BigQuery (or notebook leveraging BigQuery), +accessing arbitrary gRPC-based services. + +### Pub/Sub Push + +![Pub/Sub Push Pattern](docs/pubsub_push_pattern.png "Pub/Sub Push Pattern") + +The agent will be deployed alongside the workload in the same container on Cloud Run. It will +receive HTTP requests from a [Pub/Sub Push Subscription](https://cloud.google.com/pubsub/docs/push). It will +receive JSON requests, convert into protobuf, dispatching it to the gRPC service, +and publish the protobuf result as JSON into a topic. + +### Pub/Sub + +![Pub/Sub Pattern](docs/pubsub_pattern.png "Pub/Sub Pattern") + +The agent will be deployed alongside the workload in the same container on Cloud Run, +the same container on GKE, or on the same machine. It will +receive HTTP requests from a [Pub/Sub Subscription](https://cloud.google.com/pubsub/docs/overview). It will +receive JSON requests from the subscription, convert into protobuf, dispatch it to the gRPC service, +and publish the protobuf result as JSON into a topic. + +### File IO + +![File Pattern](docs/file_pattern.png "File Pattern") + +The agent will read from a JSONL file, dispatch each line into gRPC, and write the +results into a JSONL output file. + +This is more intended for testing purposes. + +## Test Modes + +The Agent also includes two modes for testing: gRPC (direct) and Pub/Sub. + +For testing it can be used for latency testing (on-going tests, measuring throughput +and latency) or for batch (send all data, wait for all results). + +TO BE ADDED. + +## Testing Locally + +NOTE: It is assumed that loadtest tasks have been generated as `../loadtest/tasks.jsonl`, +and loadtest is running on port 2002. See [loadtest README.md](../loadtest/README.md) +for instructions on getting it running. + +### Build the container + +```sh +docker build -t agent . +``` + +### Running File IO + +The following command: + * Runs the container on the host network (so can connect to the gRPC service) + * Mounts the local loadtest folder (so it can read the task.jsonl) + * Runs the "agent file" subcommand with the input and output files. + * Configures the gRPC endpoint, service, and method for targeting the gRPC service. + +```sh +docker run \ + --network host \ + -v $PWD/../loadtest:/data \ + agent \ + agent file /data/tasks.jsonl /data/tasks_output.jsonl \ + --endpoint http://localhost:2002/main.LoadTestService/RunLibrary +``` + +### Running BigQuery + +Start the agent in BigQuery RDF mode like with File IO. This will open a port +on 8080 (on the host) for HTTP requests. + +```sh +docker run \ + --network host \ + agent \ + --logJSON \ + agent \ + rdf \ + --endpoint http://localhost:2002/main.LoadTestService/RunLibrary +``` + +Create an input.json file with the following tasks: + +```sh +cat > input.json <&z|AFmzP$5_6!LE{47032i{3$ z>$CxWA-Su|Nj|F_rQUh=jPBWcX$egq_?HZI^iJNepy!QD}r} znhv@(X1AIMVi_5S;;VPs(rAP=bXo+c?)$Y5 zH4eABA31Mj3%Y+${qW%f-j68cydWkztW@n_I;1w+TkRJNSWj=Mi;HvMhX5Z$Av5TR zi#wmsqB(S&y z7AhKAuozja%su7vw{+q;X=%|2gom`OEV^Wl0+kGs8k7KSVq#*7(9FyX3CV9VG`sF< zYD)9Rj~`IUaI910|Ac&&F%q${@iFP02D*0HG$JOI7Qr+9=FJ;AE-pe|`@cN?kqRhJ zL%E;{oc!`RdCja{9tDHJx>b(P=#}sxb6e|cYb5Z!sgm{W#kbSsfId{1J|`uGoo$ch z+fA1aezLeA;)HIOw>vGiSRJX@<^Z~Ys|G>Eybq~MwaOTvGr1|N_~TXxWJr&9vZgA@ zaT=hVoc9MARG4u8{JI4hl4C|dj}KD@Lc7e+nO9$_Ut^)BK55XiDQM{<4f6;gng}!~ zXbI)@iyb0l2>p4)lne=H1n`6fBDCM`74$MS$?cU|C(o0WCX-WMyoU1YMJl*UBjGdA zK`(=%WLv_5mgL)Bc@m<<`A!mXXPoi6!f8|K7;nNx>ZWc+bB( zQv@i0dVr_+7n^_hu-_HVj zZn_3)xI@QFK+(3p)JC-&aLZQt>rm*Y)8gFT^ubgKv3`R?w2IjMPu?R`M8siTj#9AN!W!Ghjc+QyIxjY0AD;Gy8~X*wUImtFSgF6D z{|%3g&e3>2hPj2s$?Z``nS&@bU1l1D&uO7}n1WR~gHLOFwg^&(aS!Okb+sD}hm!lv z7#GhF0dyptw`Vjn%vz-zwD}kak;f&!qu%vIu>=fSAG)z~Oh4l~ z%@TFpYSZAG#qVqeQ>LCHyfbtHSIbqcmP6l58ZP%|L1Vmpj&ncIjlUl*{$g=Y(f%pA z+Tpy|LMrUF->J;M_+zg5;~yeVhfs7}x5Vw+o`C!`b|E4Cv;vBg@pc%`b^psirna=+ z`1`3=Uw0{RRYEc$jH<;i##qofOi6VKOeX0{-%(Y{}{(NIB+aSs;;qF4U9zcDFqXZq_osE^XOgeOTw}>KOQg+}1?FxMX zw~Z*F$jh0|{;tg~jF+0ek%9!&V2Gye@wnW})fkpa)tPY=$lQ^jy-A}})?~SLRr?Kd zp>jq9DT{V7+TzK2zr_t==U=_OPWI;(2bKU&i(=NO# zvi6Dbe5iHxbji~bsAtH04zs+MN)Cq%@u)dxm+-~zRB^wuD*fq43b?;HS6~cSJP}z6 zJ>anelw5DUP@d8#9>~sx;r#GDq3PrH=WV4{bq@ml>7Og73}l*22bnNL=d6!_Vp>2# zukq=qs}_W}Zmn=gazV4(y~tH|p-AOr>(yE;1+N`Rz}<-)ukih5iplBl1`mvw2$f-V z0Lcsv%eAu_)T0|BH~AF&_;A@_7n~G|JjB2JDIa?G*TA`Cokl`sp3k}+mGIzzHPe!~ z>B7|As>Kx*jO4YMtW(-@h8>g*X8e$;3RARSk zFV_1(4*#4jB%>m-CnpGWPSN2UZ#GTW#`x-2cFQ&6G5bc0q*RN{IVEFFU7J%gLkomK zC$|YoFt*7Wmt*+MV2u4-vO~hqj7gx8Z#zt+GmXv&3pgmxM^7;WD{NG2Fa!eS2ncX8 z-*LxN<&HRr(Fl1AfKz6EP!)g&XdDqg-Q6$bZ-qOMb&GV2ePz;2e(E*R!UFG3`Ug?A z8og3H6jGOH#YBNx~t3ItE} z&K$bar#IgI*`|F}Czf_H{(@KP8xDriZ8rx+Wo&ha;d!)0o8SQ9DRt652H)tepusjY zH_cO^bFy#Df}`Wkjdw+$(vhUFtSXQLdRf0_+@4b;A?B|Y!Cfctoj{R*7ETpYI4y#* zGKHX4x_e7wy>_N@(A>3qp3$xN+qfz?1k?{X-`yM(_m&v7TlPBwpLf^6(ZI?Q^s;fP z4bG!PY5m{4eCPgQBZEi8c^q9DAj|%j?>fiX#sknxF%Xvv?ujov41;y-e^?0@MTel! zV7wc3xwh0Gfz<`v?XEyqS-E(FhdIz3aV7BK&BY3JwcXB9o+_z?Jm@;`Ev4rq zkQf+$!?iy3zIZOt50R;!gsZZEuzmc3ac{>qBK;d>wHM~tP$fi-GvEGr&hH0`{YwJz zbFmsp^HLL6VIm;nv9$S|r<{4Q9wI*G=Q5ty`J*Hhr22Q{zg7K=1mgbXu27{Rdp#uw z1{%AMU3e(rh_DP&8@gJ$2wZPN_qjYCts(2CxVX562BaPq_Jj1cS2y653*X2?E?#Af zG3<;bI+Oval8tB2=i{_cv=zFAaJ`HHc>Cs*0;!E-`l72%1P$U8aO0!bqAyU-@!iza zCQ=n)zp6K2e<_fY8vp)`n;_*Y`2}HaXJ93!Trbn|m+ZsHJJzB`9P7-4@C*Hr;OYW4 zgQK`?kW{)o+%k_eU8+f3p8{@MxVpyDrX(N_+b}h0nsi@o_lNe!X;YJ1@M=WoM@I)T z{?_9dO>I8D+sbe*MboVixjXK;I-fC#MTJzd5Cq*ZuSpi&uXH0>Tph3OMw$??{+vE* zpVFxFb*Fn1G;1}oC+jY^ITLc9f{ek5mu~oM1{;D$-70!;W%#q6!_dHNQSa()Up=ND zD7!(o%G$dcpI`++hJ5o?eM#~1oCLnZC}x|@w(K<1rxkkQ(!r_Q*5g|+nCm{9dY7Tx z`=^FOhY!E2dbHgjwO}Om`1D$%Ma;bfzRUyUkT=`%|cIgDLGAp zPR(;O7`o(y?=)1v1uL>MAJ{W%Z7M%Q91tSLXXUxg(J0UPGsZvtuE$;q1=&6P=?*Ny z%dDAnOu{awR!!kBsU$1sh1rMFj_6J0gmwa%U2=eP>q*4ij{bof3S`QF=}yV}9p zcwS^x)JaufSM&I=JUuAT`3tU}%4XIyh-vQk>M9e>*5a*wn<72|yEu{E1y$OY6XXhYKL0!FZ9IN-iW?0oJ7tK^aXxqrgU z2==j$y1EkS3NCo&VTn2HNiX2OA|32DLaXLn*!+To-FK_LGDzTM8p%ush6`cr)zJ~x z^!E{a%nm5ZieZe{GrGm!wwUdug`UIX|FQHIe`6F^(dpL-XjqkYFoqeA|Bm@$bbej` zQY0gu<6(GU@@tG`h-+XD0qjmgS1{+fh?p?$EFO}FtQU4KX?)w2pqg&4$dzA<=0uoO zU1FV#f=*+eaz>q-qhsfZz(aztWV6}OY+hKf?M)0b> zlY_|k+vlRP*BF!mx&QV$y$Dt|hf;Jx%>ds|E}05rb1KHbV{fHBcOgYh=vO7D?dh6@ z%ZNL7(&mlvTKg}&n{9Gke?1K&hN`^B$i8F8rA&Kd?{bi4!)(=p&{pBtHklD>z<&(8()s~CCT`{qM;=UOR2<{Qu-lU7tJU> z7Oz?Nck7dH; zvrlXdjam84bCUmU@{J4_F{F0a7D2iRHvn;2`x|%im7901qey!G;)ft3JnA+=s{L2R zrE+e>SrB*76ny_{+p)_e%C|5NoT&>eIPSl*xws|K;7IUsb8`fhq`;*0fKJ~bEQV41 zae5G^d@jTLm6vJ;0nR%~CS;YMhfnHw4l>vW1vz~Bimx&}v;XDp3|8z>D zs_HeHx!jlB!73)kc+U<)9v0Dd{`qnry@sOzwj6wq&V>FpR;Br;%~0>(p%*@Qh&X>S zFNmEeB}06aIAz)t`GwEGkxZ~5P$3YaMZXbtCyuF#x!+$*e&pL*6Zo~=;mS{!f5{aQ z^Zidz@wpdYeo+xN*qB<;RowYVaEJ9L5(WX}@gMmvm!+nFb zes&0hMo6@c05LAFoRk^99}NEz{+`XQf{!N`@Ak-^{v{Iw^OTrg%w8|lb zgf~aLg0R+KOf-Jfh9Z>Uv-pf|wL!UYmJ$?V3DC7noz+MZLJ4WW1@)6F1$j2oA6<&t zRLTsBkQAyp1eq0!`5XCw0xmXM49vrj0zWpXO`*a+lYT~mA4sa7BwP1A=7oD|{@Hv5 z#G@}~vpxsp2JMudb-QE>ftEJ3mwm(K)Gq6q5C%6RPV9M3naPpDo-`YsnGo((-!?~z zo0Y|IK2LDPaa2Hsq>B;8@?p|zxL{pjf3h~k_ta>4ivFx}I3G0e=p#yoelvwr z?9!Wy_ig$2PwG}gK5lf0uOr`=TnV3xbj}>B)Cns@0(83r#2`2#FuXX}{gHC``^Oyq zq?5_Cd-&b;88aU)aRewhoGjVn@!{UokRI>#9i)yUj&bbA+WqaNjl0_``nC9M*BFVL?k1(0 z?~>!BV*-K7?aX*!FxHJljiApSKb88#ndA%+b=WVz@H|JynOCvcC~YjW%f_D*7V#LU zXy+?xNEE@QY>sAt zc04{96fpg;=d|cLvkzM`@Z6nJtC`XxXIP+v)uU1^;EwL=D^>IdzyCiAfGC{Z&*9v) zgFq$<5?d~9MLHCz`Y^rMgbZYl0gi<)-qb$+F;UM|A~-xC-9=noO;LYp!&Twi z*?yI##tQE4<7#}WbikG$)1>JURz)PbcOJFjgFoV!)?{g6LfVpfx@UJ zt;DGK*v|-R7n{B9nL$O4wjV?u!iq7KiUNWIs!xH^Lk=36P1IZPjCiXsIJN*-@@{fo z=y+WTn$Zj0&a9u6K>jW`Nd5C%p3K2(2a6^#QE{~E`gh>bOl=|BH>Zn(;U!Yw_BZ%) zr2X(YWNxO}M#$kGX9~2)HgbtqDNfn18OSXosx9|lnLd0<3AnIi1iM<)~PrYWV{2~=m>oaPC!>08F z5l`I~k9TCOy3kMO_S2;We2#>7hU8H4cx?ui>~gYC%@m4tL>RP?V;E09sjT2;rv%J? zi$-n|H*vASk?LP1P>2u8ICtu}zgUxJ+{ZwLCHzYfo1Lt;F@`%zYRsWY)Ma;dJ&#~$GQ36!V z15Qx*A*+;6f{UGS%tBM$>GOp6*cqI#XV!hb!Et&&tMV&Qm)4xINVUD?Kn{58qPiNF zWei$?_EOgZMXJH`mT?hZo1*ROFtN7aCecobd^#9t8F_@CI*7W>59>fcuoL#*_DBp8 zt&*qI>zqTu5%`Y8{uh$x@^`*H0ZmF+x8}^}`&_wYhuPo>?SjquZu}DeIst;m+9tfg zgI$|w@XmG9vu7Q1_4;g^<=?5uT>G{21$5gL=e;t;*t`-tH`~rB*%Y zNqR_BuDAk6b#LvHe+fd~&Wj+B3-maNJIod`xLk4mWP<4u4#~dQ0Mc$*m1Hhcv=s!!o=kp}JevX# z5qFdyJ37<9$RPCqkk!mMFL2;r`(6`t2aGOL zUVV~5SD$p(RJysnjaVzy?#uY&r-#+t%B1@7DH(r54BI34BNquEG~p@RbD9ESmAIA+ zNA$Csd;yg~LB?_zSa8p%jUK5cR)c?%mH1`!jYMcTRZpLgV*;PN0kYD(AhV~u6#n8# zzzqTbUF9r)aymxz{|g6PV5r_(|BM@ZB{5!7s-*ZaE z&wO0~bVjp6pVZ7ubf8PG`vv+Yr5!?k@yA^A`vAPU!vzR#QL0+w`yg|A)8#wna++Z!Q3! z@BbU>2JqJn8rBhFz*4|@<`{c|@c+M&`b$*|5C>b}+WPu5{$mbiR9egDKow=>b6C~} zcV5snsVM`KRp1)F`>#{R*RLfD9?H}UVia~qJc85_?|@xhq>-JU8t4V2qyj$2bif%a z3;2&Za70?HUGc(hsyG-x!t&mKV<< zt=LP)nupV^AVben&Lc@bWO(BzO(IF3M?geYvi{+}-DrdLv9RbB%Vfkb4plAp)#Hx5 za-57px8R%JED42fy&u1Gg4uZA>F_W-IQ=cB6!O}S0#LTt)JMRtdY>?HcI=x3-iHHT z`w28{meVWGWilv08q~25tJR{)bJ%Qo-Vio`@)l6o+aEz}Z)uD09s6c`GF(?TXg^w7 z6{kM|(>hhg@cBlLL7<){Sr6)SL_^@+ch24biCscCHaYV=#76gxdFhngwOzbgr5*im z%}V>Y>~H$OX%rVl6W9|A{6zlYz$B6Q_V30+_Er`W3VN|#fV}*Q@-QmmQ3MMpxc>$K zxN5geg^wRU(v>=pPC4^GpQiFc4++^wCIZYVra=d0!|eB}`2FP|kdAjJq!g?y44_p~ zfC6d6CQ*Lwmtz=USHd5!1~H& zViSGJc6Q#6c_iiUu@W1>+E#c^9vOR-rRZ9 zOKxI1TWhJX1gW^qwVg2&P11LIGOMg+P{2(_U`_-mlCLbQ34nlvAqcqH@6r~dqM&E@ zvvt-d7xT{R17GN$$HqzAK3J$WDEK$Zd?S96_Y)RO*m~Op^d&1f8o<>}0qsV`@$}2e z0bWR0&&2P5m(;#c{ouQe`{v?v%y`aa4Q4jLV8P}rpDFx^_O?}kE$U0EVSRUX5|xm^ ze&MNjgItIXSZ?nGz>!~pSg_-Tab-fiP!#o@n>(I-uOU)|G2K2#=0a=?Da+dRcFyJP zGrK~x`?cQK6W|p~RXQFDt$29)2==E|H#fJWK(fA%TxK)2XgDgKUi$sU7!HPWuCrgkph>LktjW>bU$YS!+!t zs(aVS_VJzc!i)m_s~pY&1o3$2nQb95IA6K_@!^)4$EA@fTS|JzEj-=6wyti9+0RNA zz0!s-T$UFVA&H~bsf2acw9Cp~_)OcLDI}EbITjwf+u4P4W+K>r8>(KQ;B&p1qV-WJ zkhu`)zAv7N%lGu7u2j#sC44f9S*HRPpjvmm&-(G>5BkOLG1(R7u%8W&Rn_EkE`mUW zphIhaT&Gx5XM2u`8`}FK&-CcSY`Zyn!u}OXz5PsIEJu6sr^EY#RAfY)$Tff=(latX zPr;?pDONV#qYe=Tv7k`y@cmQ&3M*6g<*Cv+)(PtHD4)^*q7?hPI#D3y9T*4IRT zKUXW(-MpUL^`#NAanQS={IG8n>UA);8qJt}Oci}ki?Fq|#X7Yf%aSURuu!WZIC^oL zz3Pzt9?QY@oxJh~@fkLtUp+Ozzkf8_>0R|H~;NH;yy+8x+v zAx42Cny#+2e*0V~EYGgk;MaeC3-BG>E=Mw-&o#fT4O(KG-FTzcSw(WSU&JOae<=cg z*Ig<|Oz6D@P4{T9o$RY_xq1r+0zks!{ZXLZYW?JNViQ|s(30cMhO}CzGs)G*QS{$M zdIV~vINja!NwIIe|Jcc(DXfgnnjU7d%^lAEh%GwVE+;1>CXR1hf+!X+@tB_o!x$MD zN;jVI8K6TFKB27^47OC1{hu-awfXUP_w-DUYDduEVdir9KPYQt;J+~+`Z64dj(nd> zbWxradr!4())|Dw02W z)aWTzUrGAqv?93?PqF~w_K*Fx7|khCEi`-^7DfVY?P@V$SYJ3<^tnHuQLD33mfG3n zhl&M0i1a1U#1^}0xn9R}_)1GVhh&o)w$Kxwln6#8;7C@ubv+>sul|OVwO`xtjlTVu zuXk>$$zgz4QKTi5Wq*y8DvxvzP3+kxWhB&Sk1+t)b{-hO>A}bn^6aKhY&v?ghP!i* zj>z%9p#09N7sBU%D0wcM1meM0Sl6mi%399j4Zx`(*Kc%+AI*y8H;Yb8>}RQJ4FNocnr%tZ-NWtC z6G3Qhgp#Q@1u9nKOvA@<{o93BAOH!{PK{_#^EH{rKmB*M;|S|#KpFD36sT}sw4lh%c}*t&g|_wFnHbSag3JWoT{kUhbY}wx_`< zAAHt3@bJ&?>)&_LeLa#Pu(mH~majACcgY<_#O)wUOGUFq3Lc#LQ#i6aoEDHxfaWk` zD3EV$=}C-;8P;gti{-G~PQ#aho6Mrqm#kwP zy*pjTl8Z2o?C1W3UgZf~yXwDe)ef-_oq2vqP7W7)xRO%loS2+UrfCg)d|(gNu|05F zZU^%@{DYb&gQ&jvU&Hegr*MuIn%#GeFQC?_e*to9&@sC~18a-tUc}bO8N*L8ul-c+ zU&qFz%$ll&F9RrNGn}x>8bOmI*|y=cbCJ{T%c}lY zohbMelCnLa#zRQ+t+&sglEnPHYh;AnsI|rVYO5E7F9dd|I~KERnT0a%`f8?qwV*o$ zR>-%=7mwlB1`XUTDS`pqDg?mUBj+&Iq*8wU`ZcV{)ncl`fE)vly7O;^!C9)r6J3m$ z@dFgd;0H%Lybkm;g*+9|l3r7v8%G{L-0W42m+OIe09uog1;Cf0;Qkl|5lk~(1Kel- z7kas=l^(|fYLZm7s9&{Nz1%Z%V7CqVC)&`U2gv7mWxvJnb6 z3x|wN|Hrc-fHV^J?*d?je83fF4dVnA_x}FP)aee*FLMQwHg@>=)I1>~j1I%-Z)D3$ z(N7dleAmk6trrF1lraD#GxGgIAW>Swo8z?29|b>RS0%>o;tX4NdEH*W9@0E6Hg10g zzTeq)935E1v0M()-wYjZuXM+_B_?;TJ`^`Ti@$GP=vw?yigwPtb2)xl?8DAph?Gbr zNA3Nbs2pI=X<7@RfUUpuMB?joF#dcV*d57t^y9~y;cT!$nuq{JHGvyP8{#Xn2fzVx zmoR~O`Yh_0_-pw}$I*g?8cD-I*3&rO`-6i(mgy!wf!{&aqf9WOsZr7XdM^+q8V26z zXinh6^@7Z-Nx-#?(c`Rjj+QO_xqV1B$fx1#bcmxLD)Cq;<|>h4!afGR%+#MWov%M@ zj?dX#=?Y!Z3E|71(QuZ7ohio5)GTlG1g_5z&)Y?)6G4}2P7SD+8Z`Z9Ps-|l#O6wj zjj6WB{jFv~w4Zw>+HYdR%EtYz8P+sTGn^$WBq;v%DX0v73b&$4cgAAAV`kD}exAbn z^R?!71Z!d}+(iP{N8ga})&!qgfRlV;RQw1I&Px+IY~IdZ7&=Z{*-s4`9v$sVF$pLk zjwbeppO|BypICUe?Mb(rQH?I(OWAoo{wkA+bB1#Y(Uw@ZIRuS&`~L zE&J#$-L&nBw3`FHF8S-T`<+m#8nabRUZ43(z4x~EHn2&r0ZN1TKBMuLOB&jl0Uc;j z>{}zYtmI1E3Mh6#-RX8VYo>}vK=%DfK~Lxu-U2gw<+|%CR+Fi#5`TDbHs%Vz+2lw2 zkZyPNUP%LGgasm@@{Pbq7-%Lio~klgwpA2;xU$$^?pWSAdM<@WR4HDSW*~Zx(6F4} zHtmwp9fdUu^g}>#kz1a1J}2hKk#&&R&E@g*J>0wW>B`HT$2-Pl{h#lKMx_8&~yOb7U-#xf^;Mam9V*`KNV#5ND8bG&Mvolm}a*B*hVnqjL zxkhLq5@Tp1?=%Wm0KW22C*0BwiuV=q6@2@N21dwa8P{kEavqfQDHW{|C!`WcYv$bd- zt6^2_Dy>q(LFB&Q$3MNvdp)G)LAeSB9rxz%4(1IPHv0Z!8S5Pa?2gk^4RsBT>5!Vb zwSP{dN%i&50@CalDCA)FD2eMc!vOcE#_+AnsE(KdZ2ctVRWukWGOGoXtwQgp<~vCX1cOK2D?!UPt(hw zEgW{;hvOdn-OD-i>b8f`xse;fR>P{5p-TVoe`9c8+;rc62>`>=n9X$30RhmnOJz(P zwf*6+SM|1^m$>dia%XVKeFRl)p#S-nz_Ss4tld_XgSqR$skIr@P>ri`$l5oN_0X)a=rhC=?j^M z@l_%%)kxmCt>iKD6Z!hNeuZt``*7&gxIOp%n`VMpnP5g0qwj2-VuGrsrq=tnejsai zlNjJnG)zgeUaxUU{VwKbh0j!(REg$LdA^8=ppA33cvQM3LNq9uuKo=SO`e?ztvhbP}ADn%li`bE3qS8P= zYCTIR%Fz0;M@fd5OZew?e<(A3bMyL)v}98>dwlYP3Vyl(;{&3c-@D~+1zg02lH7gA z?)}(vZ)}Wjh0_(qI)v3^Zx51x@h1TdPN7VyU!f*~x1?{31xiJ#TkmCbKQBa}fytw7 z5)Zf$?~}^rU@VDo9M%YFM4yzN0HmuqEBn2y(BDP@BaxBe_N3p^RhRKBH3At`)HHE_ z>sh0CqTlti;71rvX<{Y4fj>SR_-BG^W!Qd9E@-RwRzr(~Dfzz|iteK&m8@j_Gz3yj z;j7MB2M5np6_qmu{1&|ydHLE^7qWzGj!$Pv*%;B7szOusd7o?(!!Kss4FqLLm?;IiqrFfE`bMI|^V)-Yz8QIec!T2hPf9lx z8N#sDl=`!1V_WpG{L!b=s~3o$lVi)>&#Xs{-lfI;a3$UDP#hZQa?SuPUbMTOtGWg^ z7}1>hY+jvie(}4GM1G5TlNC79oy!)h#YEnx##~~q3$y=ae!F~4Tb?`WO;4u7V-lKe zU*fH-=xJ;J@}CfDKD+lJXplh4SKV@hGyo+8x|~Vw>w^u#4`CTIGgLGm z0f}AV7z-fMF0!MKeHXlNa;K5sR2_4f7xdqvKJf@4D{;ZgmF~ePbu&UgGMW2`xysK9 zsv-we)Bcc4$qG*610Vwm@(m8vg!;EHnG}%$G^Kf->xxKqB1Se1zJabnTb16l7TI2> z(fr{(yC2<@>qh7&C#$!UZZishUT}6@8G)t7%k=(mbaU&k0j(0_B^rz>Lk9>*9C?pX zfz^BYL*j8I7hW9^G?+-06Q&jgStq_#2fv5^mrACx`QZDSG!(W4xFb1WzPiX zl2#s2#83$v7%|Y#xYM-tb5`v!=6pgfY_*VO=rdS=9A z)7Qnw1b@;0@hw_!%6M5K^|k>3PysD*!2y%2tRg4zXT4JpQnsrNr){K-$q9Ftk@=E= zVYjjQh}FcLHU^xmz1u#`9!}giwckmB7X1k$#H?}Sbo{_YNQ8i*X{Z=#ojnGiPn#1_ z&HBw68SlkDSz4X4VU76n1NvLChUSQH&~!MuI^}c}c~eA~Y%OG;UIw#@927R<=4&r` zTT}}3%7AZ<% zY8=Ype+d28L`#Xkny=A%c!Pjpm_Y;`rvETB!?_M!!k{q9?OOml)97v#Y{Z_Vv5D$}iDL6rz@4g5K`mG{d2TO@~@# z+FCAFg^Ybv@0j6&S4ydz61~aRFQkl9W`Ntmw!1|`U1jTG5E6=JiOU8&s63k3$7_c=x8WAE;{oXi!_)^h`1 z!Dpv1SSmj(X)qv6N7W8PnOKcm>EUUgLNPnjB7;u0^la#@t=@Il0oUFZ@v-m~00W@` zLlCMf(^%g^K%wepfWdXFlGl{7QLx>~-DO7ut-pTD3tD;@inz3D`;EFm)oJfd2*ZZ^ zBj5DS*B;Mtj2@h&@K&35S92D*uEnl7N}Wj$rFZ%4+;HZwHnl*sC|#54Zh#US*^BSf zgosRDAj-W2Iu{1qeGBIl6%^F@0>L22Na3MP$h+Y^SZ*%LgfW2&(fp)7FPwpHf z47|C6`aE*PaE?Nw0vUTZKq1n2d-*r!7j*lYXZ|_d|27WmKU>^em=u}Cemnk+inzp= z6N=YOb3m!|6Glr5zZ%tv!s-UMNsa$XnsZ_jm%$i-SoZ+le>HE@|xt`ZbN?_7|@GU9ueTgAP=l z;zUXigZGIy(Ame_Uq(ySb{}Ufeq_)YyR&7|W@%lX!ml*`?sTqE<8x~IGPu?P%wqg& zN1fZQzjjrXEc>ouhIImrribvf9ibkD{b zst~!oo)4JeU*QZ6u{MaDnI6fFb0wrg+ope8k_Cg}=EPks1>}xLt)Iqafu2Aus9qeu zL<>Z_j_a+AX6$aBFSenSju-lICrL>pPhDWwWENpB8T<# zUKwc!{pgB%zuFV2#Xw6dg-_nX>gJlVEqXdY-`sv+2-~RCYO!@gAwH5jq7IO-G3Gb) z+0^OuoG~hDb+Wlxy0eKq}!Z~vejF;(V&#wVM(;Pq*k#Amu-?aF_L*bm3t8r=7f5{}TopH*cqOQWG136%Q4$>Llzb=LBMcjsx>WSLR7e}YMc+bO zlz9@78BQ6uKj*F9xmb}t)kcx0bY{i@vvo${yUu1+$rnzG&<}@~RrijJHtnvq> z>RrFDF~E!nS}KEzxiC;&e%f6K3=pSc$)0m^f2OJ^<~1F7u?XOtHk@ z0c;7V$^ZH%lLinr@z$1!s-@dubWcz6@-K_`@*`cvCARWPzka@tF$+IxdQH-nVbIxH zs}5lKP@Df;o-x{ezv3j9yRxyE4`h&YKsVWDd&&&$MbiJ~0xWq5P^tL$Ck2ad!WFk@ zT2%jIj$i)gB-*fUnuxwErGFMLc(*b|Ca8O_cWjixzT z+M~CQMt%Ugxq-vP3v<8rgd4moRuiRIM~}$+Amfi+tTHU$Q_LbcJ{YanFJ{bAQov-& z`d^%_3Kr!e{pZh&jYC@A_>Vg{f4gqHI;LGh;>dGTK#lG*z(~P{x1ryN2h{vH$P4=N zpJZqVSqOardtMOv|9WZcQ?v2)|MyQx;f^KmGP*aveJi8#8dg`f!vEDn_@00z51^5=xX5q+ zD(C}Vc2-vSum{z|g!<>N6p<|RI%F`v{qe8Z(a3$OrbkO{%*mE%+?J>anVqiy0Btdv zDQGG>43H+sHZ>Rf+ku9`rxHB+f9c~k=1)(P;oe{@1m#XOA>ut|u^|J7k%5-R`x5us zx=s(zyRr7;<=Q%KHNHU*eSqsc9gykfi5sJ(3Vj8673OXH`gFJ;m`swr^QN8>oaoBR zo$sysTM0TgI4s`6hHLvSTK@e9)y~EOhF1U>T3)X7(b{AX@q;uSqW*1-6RODO+wnva zGQ}cTK^1YwOkvhi=*nWC8r@2p=VxVAdRs&ea61I*fEm(MUl8jV`97m!WkoH6UY!yXf}4G;B0dzBciUZcDkb^xYJBF$4OCQtrtY$?&waF zm<4>FLKJelb|>w{TpnxJ*rWHBcFDAgxU)HIyZi1XKGmje{>VnX*L;^Kn2i*ZTxGS3 zs&(CQV=qPWkP$0Y2I)4f1>NsJ1$k*&2{>$Pn4GckOL}K+*2j1t-vixIII+3P>ME5% z+UEUduPEd~O>q=IvmkO91|1yRZ*6jtJvd-h;3p z2kOMuE6T41767XK+#~Vr3<2Vc>d892Xjcy4Xcdym*1EyfhkA1|&9*Oq$mRK<}^ zT>K}t>MK<2c-8%<%f(ByZ>9dL%3@=-?|uJ975;c494EYL9Rj_8W6Cq#)G?{kd)E}~ zzxG!Hd;E@lx2b|DF&gcJR9T3@`6YN^rvb1}hh>+LY~T^{tfQ11wGC<$k|zIee>G5U zY?KBd`#E4J5snYNhZ3})ZY8htcyFj#|FNUM-fgrB3m){(P6QZ-kURY|8i59u=LAsl z9MrA^ZCCLz>_0{(;ASBI1`kz$^6!B0iHyNNrxEBerX2ywgoFB(0ACG}jsG~AfSb2i z{@V|Qa6I!KJN2Ja3b;8}8AkuD;KtZXDZO*4B3eU9y|<)x1823xzbIKk{OI~KsZp=1 zVBf`CU>Q{~cT*M4C`;Nr`aSQK@1rO&eHkn*dGle(K+^y;NgLIGuZ_?u)~=_`Od!sE zy;WT^t*icHdOd^Y8)9Tsj+G$_bdE0A;o+1lyx5}P$o|1$$HleXh$hv1OBapm6;z8a z)4U-Ju(!_;I#{@2tS}vV77CXfAFq`F(`xILYJ(ByblKDkRg1xV4UJ*iR+(fecgZ3~ zGEbXe22lhk`yi^zzvveq!sD-2G(KN*OeNCQL?#tkouWYBVm%g>a#tq6F1eCFgNMK% z-a;D94~;lU`dn+&bj#Bc>&25Z$~0^EZF#K&_M3>iP0`PWLUmuUpt_3(3`c#;;`>S{ zbLG%C0Vm+}IL!WOAurEl+I|8`lL_JGTE=ZTUjP&k82DQWeFC$STjRL&oQUyL*^fuu zc#3*%pq-4lLXOVVs^5R(fl2vmbmU8|=YoN`<@|JIr)p__*-CrpZ!RfO@@ILmLQUjL zkJmVIPNmp(bp?t$!mX7%+{oyWrH!Q^d|7c(Q%AUQzpZi zxRKm^{ZP zhNx;(tl%8oMhEl|+xF7|FBat6U-OANPfK31a_#;@p6HsE#Ly69%>x0zN~b#o>GRif z;z<$ZQA%)qR2*Lmb{sT@nNW8X5pUIzcug-7Y($lKZjIe|iCYd0hGis`z@`w;jW^%4 z^_uzKiMW#To%L^rDCYq#HSF;FHaS5JSEi zo!jL>xYx|Zb?nVT+(=^mq7E!en1@I7i(0D{gI`HsqTb60#q8Gi-g$EktG~v_Mrbek z-h)r?5}E5u2b6g}z7Q&yJmjyUNJa(e%HS&-n8xP@^dRL-6iY!pwgYTf#N zUw@blePtk?ny;L%a6?Q>7Yp61a2;){9%WIPsdIC!u-D@-aW_YjVY`-T98aA&SAl1cD7Fq9sxyO6n|4 zdR1kH-+1@Elpp>u90sYp{i!h^UPs;?2r%E?!>!M4w z@8kY73cwR(+*|!ZbS^LDDgWXkKp1KPC_S9Tul`c+i?54;%Y{xVWv83HUbvKt2*|%4 zX_mno6N2#w5MIAmy}x-F8vM3Bi$quEeh!2h9r~wJQEph+*Z}}vMfkATZ(3Dj%+{9M zhR^49#SBHHNJV@}74;22KhFI=X#4@`ytxh9aavzS*$Pq5msZb3rOx}yIbL(SuK6ol zD468fUC$!GS#x*jZYIg<$5=1ODygNv|HoV1!OxSk>JCQXb34vL6_{-Nq+QC(?`lox zd%b}!nNsebLrq7)Fu!p#*xCxR0I{Ne``;1m3?T8&g+&4uVirFz$<6-G&a?uH2sR*T z$Q4on8xRE+J}@{zrUUr!Y5(MlhklyW1IzpF?Nrx;)k4)mC&FAg+PuHigt`y9-465 z&9+``HgSKtdeAsr>BLYLJPUR%O!^lEshWV(y6jm0ub{Dqu`To#2b};~5x*GE54V6^ zM&hl;4qmXo??9?@nF^9wPKKH_$BAyP(yxI+FDUPlnrk|Z$ie&kAQ4y>;5f4ccp;6sZukLYiK>n4`r<>B#vJBkkW0QWyZ4g&{4u-YG(GW1TZdXlE6^Y3!* zqsPwuJPU;N3l(X{`=}STRMynhJo)2bHy$f zZ*iW3)8XxCe~%PQtpDSD{>t@w~*W6k&SSAawu;~lD z69309>?o1{j4FSf=X#N2?bIV%Q}HT~mv49O@fKs0d@0~A=LYNDI`&tx&!R9so}O=% zvfm)|!qw>1Q2~ppLV0rN0H(l7)lJaT{o0rsh%K~zz_6Yh^XSG=o>3Qda2&6>*;51| z7ZPR{8~U(#O=%htWMN_98la><2yIusenLHG9&VfgOcFqlATE@S4wNd@@`OOP*r7&N zz7Ln$_y}i!&85|{3J87nt?P&y)doZWlK%GkOX7~v!CZ|@D8{NG7js>&$8N6_jE|lz zhp^|7m`3FcPBtPwX?%0*3@(T)Bk!Ky9Nq(*Z!^d1_y@V)ef}{KK7KKW6}#gS@Gbk! zR>urnczS@}64pz@YgW@-O{1OYV}R>($8`X%3)IhSMi&UGBmiefrAm(*(1i?lk_Fx7 zL1OZ|rxP30nMly>9j=j_sIg-y&}EdHVh{ZnT^LH_vj*8B?6Fz5>!H_OQQ()}eHT&( zq0_!V`3sq1+CFOjW3EBX_eDm`=@Z5{oV2tEAp6`4x{QBn2!tWPj4OWE7m5IF4w>)uJIC1*9qav@wIIytz*3)lt1%$O47Qmnp`@m!9y>vH&Y7y) z>>^n}u6}&D3kp&M)8-72fVOfj05JbbUjSEb3U9;OVdt4N?ld5meOiEeAr>NL08$Kr z+p-gqQjTDHsV{JED!xSpzH1uCT@C1CB2VqNZhcp$6c|SLI8@h7T`uhNolxc$dJ?nL zOy~}f6lq?L{0hf)nlp40^O{Y^9{KG;iUv-0QivUa0&#a#)`V+ap~0W?#ilS&Z@CY+ zH8};U($FDhy-`0ZbsN~M$0@cxY%0mg%8GXRJ|snVIc@f$USA%}X*YbZMA&%N-Q{v+ zdr~5Y<@Az)jt((a@EjSiC4wUfUvGf*pb?(O@0BCsCk%L#RTaenozl_7e7^U5fPski zC^F}PEHjz|@*PPinv2UGAB;8&V};e`C=M@~dvPGa+y~#q;*8na=RTv8oUVtGSNx8C zDL?>dxMRstY+GwTKVdYNg5d@1Y&aoXcZ@Ds`)@})$=Oj`P(LeUGO!b{6Z6=WRrlRG zUyZ0~_;8J190Uhx>`fNWXb*0WWX{08=~}*TL*DH`z6G>4PZr)+0x_JmquL@76y&Xk zw0*gnbszYK>OxyOykokU@bQD_s=hMpYL^+nS;9Sx=S$$*0n(^&J5UtcJFRs+J=DOy z`Oyr>Bu%yD$$&V9z&cucbRxEkNy4L${^}b4dwQkMy2%;v>S5yiyp2)>1dmQBRTeh| z9-ob=vU(=7BwZ%J))*WY=hrpJmq_QW3kl4Dd!fksaiCF|M9x|SNRG?2YjOhl&=fUE z|L73zFMtg}@Canp9Ztcl+rBt-haN|Z+@A&w!O*d?DvgNpQ`7CDuzHOa#<5rwb$eMmJ?$AbwL zshwDm<kLFlqoE6B5?XQ=Her(^Ofp(HDb~#=X_o-A_2KNY9i2-ww&Ca)zPIXIis}N&Yg_knwNQz{{yEc#f-q(&f>bpB*QZfXA)2pu4FZ+pv ztge794x#2vv8i$E{NREBn$>wVYOH|!X`UXQ(btdV95ds1<_;%r|LKX)Zx)9(DZ)#l zv*|Nf&)5E|>e=e!pS+J`$o-BPv_^f~zL(he)3;X4*+i98eJv1cz-Xhy8MgfDu@-}sDUl@Np&Hq|9=1&!xT;C{0!28(mG7biJ*x!!L-X;W;9^gk`&-Tdvsj@NG|$mL z{Dm#Ap#rj1wir&!J2PbWqf)@D(R;A|?fQ|o!*M;ZzFYvzC8Dli{maN0UAq}O$*v7jsZcxcd$paI zo>>ejfwP)#9GC*pPb|D02%yp6na7HSGPhr3@bc+zZ7`3#Qim=#c?0qJs zDf2;kEM0(5>Ilv1+Q9uIe0h!Nn3Pd9zx4r%fy-J{1{~>U*#cr^blY~dZj+EvtdQ}=&uT)vk2(+=?5GD}cc=kd*@C*Ws?$umeO6bZR{ zY8~@To^kbwf5(L|6Z^N~CarQn)Yb!Q8^qZr55;1M?5AHLLm;i|!^aKor~9w9ukT?&d>d;IZHfP*aS!cmG%L;RHidDf{h^oxI!b5{@#n@Q zz9!DujavS`WLQf$p{1K~3C!{kjSTAD+1*z$=ra0i+abBQFJAEe%=>O(ZD&#VEwr-pa$FKrx+76 z1O6myGO0IT=8m3!hjxr*=FX>f7*G484aCsZ(4>(jWI*oHK00meJS zodBUkC?ljpj)5N~M=sh7cA2ExY2%@oKaNd%wqPYUy)k z$sys}J&ZxuhSL|mZH_P*#bvk9Y$!-@`s9<_z)t|yf4-%et7QZU~p zvLtXr2qx3|uo{D3cOhndSjE!Ll zi1)aj>6uBb>k%Y1-pQ1ai`Wo7{nqiS(g!Z>iK3s5XjcM}O*~$DxdqlXNoG9QPvIf<19bG^u(SpLX-W|N|I-vVQIXBU zGd{QdvIq?Zs2JM2^gEO?^EtZnqHZ88f>nDhnwG5N{$9}DHSE29G|On!Lv!9(yV1U zJY;5s5`}bF%Y9;}F*J}Wwa{+OA|6CL%H$Gr)rOjbZ~iNFx@#ffK=NLe z;q&iHQg9$IKcIv4G~~G1rtm`$nvnxp=#&KM?hq{JDC z^WB@FPlY+6RktIbjCDk7^k1Ej?~dh_zS(QmV9w%1M{;%0FlC3~!4>2TA#*AI931R! zEZG*sV1k2?_Eqifo%mm{dMRFg6$B|st-lIJiQ=LTd7<6E#cDv9v4dD?ASgd_3=D?~ zM9=V%QBWj8hmZ<<=Ds8?l$1${8f)~lySM9#*2sOd9CYa_iBJCs%B7QGFyfF^Xf$LEp95!Ul<5f8U=ksYI z7?os@)aw~lWWpvU3t(67czs8%u)G873HJ$zb3xzO-|Wqa(Ap(PnT)9qIt$qBE=7q}J=@R&=0yZ?L4J=f+- z(#3C5O$m|p1d;e=Ilkoi@7C!YJWO~10+;7YI8IW=sbG+t^atKsa1R?tSeeNSvlG4Q z4TQ^S2(HEEf1kBcVQ0{R%tYXwSsIcxqg}J|f_L>MDqNqeRS_3FNgtnZPR@jJpS;$*5vBxV@m~y=~_Ln;l9SE!bMjBF9OjA z=!MY75x&k+jjX+*y*fl;&gM0wZL*%JbIbXyTcxEN$34LbDh8-n{rBuq@4GVFZqy#H zKG19}cshM%?X43?~cp@$jOW04^FI;3(uZqDR|}o};&{j`v?3?qK4)&C3dOjZ88DU6NP1 zf7agQFYNfnP=Ob5|NXz&Xs*fcY=Wdjwvt+^DR)*k-{}-`o?5e#CzbPCN-{+{b5uJ+ zA>{jqnE!osm9L=CRqPb~fxRZk(2l74K0SgzgUU3n0!m@c#8 zz=v(&|KG{kT~VoX`jV&Gs7&;PIhrDKSjB20!TZ<84(KdN{WofKm4j{SGVEaUXBErM zRI2^kw7YB;hJug0OosyA(qHJ}`z-h6d@ISy>FmB7*6PwtSeOXFdg&Sg-|e7N?!R7q z{mMq^+n`Vw^y)+^oRyT1VU#VYW;<(C)7P%%1AOY8JnD>siXq`eiKLJhrPS za1QNbzgRGrOa8av;a|G%>(uFm{VRu`&%8y4%td*_eX3Wdw5b{oMYB!$W&GI6UR@>kK{gjaZ z68=A9fc;D}T*wX8xZ{Z#$a6kptRVg07f@*4Y!?!;UvG!f{s|bk(+ADN;9=V3oSbrm ztSLFdoDPRe$SOt1*g8{jaM&TLiLiZ9Q=u99u2J+y&#>Yn7nQsW^;ZL_>FVFG0KgI} zFMAQIAuD=N5!O@M7PT^7F# zNNs@xH3i5Fuvgfv&RY@8y7iL2z-}F6bK;zn-^=ygKk2=i@Iu$^{e+BE zkSlQHve4;NMR{e^;VGB=N|OVFt)reQ4h8Fb5vTn!-$I>PC?12xaFWL`?t1n{ktDe! z8trtr@5t*0^O-TuB2YA9vc_0sAal-ZggT5YR7naO`4xQU-LZgfqjh}w7TSUr^;^hl zUNWfokiP3?_o%q7K*WfEMv#&1hZ<_3{!XHSU z(RwbaSa|#NF+3J7kit;5PxQ}ha4eFC+WYN6)Suz3&@l7ZmuL0 z_!Wz2_9_LK1wA+wl23OLd{k8fb=UCTn25O2s~gZQaNJU0$fq48ugMtzqA*t4hPI&TuF;=-|YfLp&eus@ru9O;-mO;I9?S1I_59REZd zOf{izNg_*DudND?7Z4@0)*XU&9_x26@iwKL*^u0CVblMA9(zk(`lR3d(57ePdt<=} zjD4QIPsZZ~D8zgRIa#{Wv8?3BnwYDwV4Mz_u=_a*yzs|w;Sv_Vs};{dDFJKQ;-{2| zm?GqUkh`Mg;nBM7;Lxkz>0c+nroE-mW6PI?6txL?TGvFqU>C19r`!vOpvptOhJ#@$ z>72XxDOi%ORq(eDw~k+$0p}r=+Bp8gN*UBkkod4#%%cvP4iQ}gDZeCu(8p2+1F z9FJ8qOq3#R?0rP1z5NKQ_PP++92P!lcSYn6^PFtmi{JuD@?cD<(jsJdM%cHIw-8~L zkr~d=!SJA^Kk0M5Z)fcm>%b>F*e4>Llk0};ApjDDJfF2zK!iptPx0a z6+0iOq$2u3F1UtC;Q~|`8T`<)Vu?nDoN48_lkHYwtxVJjPOgLCZ`jHXV_KHa@3B2x zpv(QN;6`99=UyS7n4l62nZv=*#n9a6%=ka3ONf1>-*UT#$7eL9#efskc90bOb_ za5EY(%iC+n_q*LI-F5)*V#V_pY~SaWFswyrJ!w623G#3{a5|LH`sgO~=SmQ5TkKbc z{j@7}D8aFT?a7tKlj{T)0kt&?g9I@XzS)f@8uW9uky=||`$hemX6p%YV8oYY%QFMW z=qqtT+j+nJo>tP5#1)fvlXrde8s0)Wu^#|~+Scfa-7rs%Ri{=INWg7mi$I!sGEXc> zBexG!hk3qm1eBYQ9{Z6`0QAo+z+Bu1*_~p5csbu+I{}vRXUd;ow;)M`@3!(d|06d+ zI5n`?^ggbGUMOz89UB0V}3)C%Y+c|N8*YbWuLUtyBfXP|fO=S-Iw%La zT*k|gvFkM#vJoh0lg@!;bED(cPyGU<+zjsb8NO4c%F3=0i!-}>NTWc$oFs7E`tw?m z^r^S^2jYg@Jn2HEI5(;cw2@Q9ZZ~Z>9Ve2Qx`jVB&AowrW)VPadTB%hkcGXj7ObK> z<`V$;HV6cm&nE;y;=0#6z%`uiE%`8xWQsQ^J2BwHTp*(!@`qjhluq;)md*Qc)NXSz zY#Xb0X5W*b5V$g1rhNH;jK%N3p3)W(6O$p}APNzyUEy_DU7q3+e{}iJg092q5Gn{&k%=Zwj$-HW z>LH9h6X(yX4Td!WRI{ChSZPg;3}iNc;HQ*;`tjm-VnE zR&j)`J{7pT5BdK|PuwJt)(aWO)l+k-9FA;)!lgEYm{G+>#tIFiGH7HFr#cCq7x?x zRyE=XVe|66qjbpHD3pm+ni)2G7e$0@=l%tiTlHT%MnF4|7?>j9kMIRJuFU!*wfT4` zxk;1fYvcmA8CEmw*@{hg)_yU+3yd*yy2UT(8I4&hYDx%RpuGji>~b)LR)xkumPr$? zz=^v^nkF0i`t`2>c{^cobF_E4X~tN_VL86D@8}yncoc_pv$YC#7WwMgBlGKAY9qbe zhMmJ32omUnlUfL@IJ1Q2HVEI+$LJ`p>TG`mt+aM1qVxZt5M^8P#i98vNTDPskx^oC| zJeU41#ix1=ZU!k{^O0|SL%EGhZSW>IFn#xnkS zi40Ti@aFcJ-8NRr&@@)yl9+j4i7Y}Rm=H#wwkz%*v)F1_uoL*ld;OH4ZvPs_8=L3l zD543XxWqO(yw&R$>!+UtllRd%m;hGKJ+F;T#j( zEF-s*+aP1c5u#u7fE2bBO z4%C!Vgs3Q%%D)M`&Ny)gpXsUw4;$$JpXg9Ie3gE_F_8)!ga6ZA_jPdy%@fs1&C7gS zyqN?n9K-nSl&GjbPTEef&lXs!o~Po;t@{(q6fUEjJpf7ouVVj;eVuZqjx)N)+y(?Z z_cJ>3t}h-cS>_;MJ3s3q|Hd4Yw(8R2A_RRG0tYlwnU0jV`VJj4f86V+V`f4a;Kh4o zZTr+Dw0eYKwc;JWBxl3h7Cmp~a81BYYQHC<)sj}+9|m>dI2C_F&((Bp!sV3nd4Xd8 zX>aJdG{ky8E?j_8%0JmNEkeQk4d4-VRMmS42arl*Q^=82aa^T)1qW92_En6N*gKKOv=Qi7R2nnR|l(sc)cnS zFg7wPq-l@~F?A`k@8X!FQ03Ez%FLyNy$7Fj2zyvisDIf{{!+)#uGstba1x8PJQxJ8 zu7>I2eZFbjt(b4|IS41wy)ui2sXV!EuCLv7L219buHR05FBwaa38oil`H6+`Y`03J zAqJ6wlxh0bV;6Zigdw_WxD{F~dX4>SQc7fY7kG*JogZvVNFvHlQ+{BYsk!-g@87!; zaIeglCaoOo45esJDdv-%VkVil_2OcPdjPVNIrz{_=KtME;MRBT8e?m~?O-D5(DX|d zUl1f|WRMB)mIh{-gD`IfK?wT?@M*Y-s~b#M3oPxLzCqLtN7fx@l zj+l~mNOEueaMJ=zYOtcdF+5{rbHO|gX> z(5*}3gp@>haNhnM$>N`C49a6vu@Bc=N_MqYQOw|C&tR7P3M`%qr7kQ*HJ%eOq@xu@ z&Z4dt_7oy);#ml{qr}xET;=?V-=)-do>r%fC;6maH# zwu{0P)^ivW!=^G13eyGNLqNemHmP>@{L|0B9^L>Vq&mC-o8LRspN-N^POEwhUpLxe z&UsO5Hx1nAJgKT2+c#fqNB60^SaDh=mWbpxYlX1P>>l}HCxcDyIe|3O*nyJd}_d6uhbsUzh+e10rBiz)7n=M7?lYhBv%) zq4Qj1li5ZM-$>gMrgVEtBAyq`mbk3q5waltdE31T4 zhbE!`9z99$Jnezms>&A9Ou-oq{h0ilHqW^x%$9oKVF%V}K8)uDPzp54XBDTxv2_(C z@3N>X*iuHvU~QNHf?mlDF0oLvy;~qZ0|&?C2NG?`u_Z)H$|3Kd5wfL@&5o~Bw`4+| zY3R({la^H1VqwyzK|!^33&8HFy_s`@kV$KpubTz5mEX>k&vyky$#v+YK+@p}tsI&- z@@p!}x1b0oW5F@~-Cz~_@n})=GbVWsRp3?t5*Zs^00^KnQCvTTkxaS}Z2M1vipBfA zszXb#^g13xCNVbn+LF0(aV9+6-NNCWKWKt5;f!JQd~At@93<1s*p%PpIMK!ln#W!# zlL{|(Su}67G_-Q8_#0r8mBv|$HWb@9>Eu(-e9xI)x*`DiL^~&cX|8^e`1P^p3Z?e zSlD!tYNK~TcmHi+i#L|_;x#++)JMFm()nQ&!;n9eUQ9e-ubAu&-dYZ2U_73gkd7sX zv0`@6Z%4;^I$>^idgt2>`Gpt^8uug)om8NWHr7P41@skgGW5hOO4@zPtoP^}4O#wi z70LWlgd|WcdNC-Yx}BHQkWH2;NPt>ci~wZp@KqnZ_6l}qhf&t8`HYZHW@gT+hr~CwXm$eF&T5P zLgLM2QWeW`wXBZ8R(*+T!n`I^&zr-BG4I*kJ!B&6+3>4qxo#@?0s}t7 zeNs;wsqdm!afA+pS#aN7M;sb&fL52D-t$7LFtkx_^LvR*ln7Eh>A*Dp1SbaZEjai) z@UR~LJo}FKE?M#n>+h(vJqjJ=ab~LEog<81{mP0l22D1S#?-@;aAmMNV8zg(dn4N!rHRaYGv}jcFb^VP zCLO+{!JgNRWwWsO=iQHs-zP#12!@Q3(SW02?yE&5Qu$G5LM4umFs zi14n#a-KMiZg$spdDT4d+7A<}{CT~M`aJNr@p!CwzcLS#OO;6DQF1#OEGl?Y7g@Z_ zcaJ_8u}hiqGINbOYwvJ#d~^(rL}Ho1ja9T>tup)_*Wb;^xog2CnyIanI4|OxP#xRF zOgp5}%$@EQb-E$?HTg)WQq@Eqr4 z6fOxQOoW1+AM2#Hu46-LweB8%>+yo4)t46*2NiphTLF)1<*&_=F6GWX$l?# zie7ZiE^_y<{r8f7ce1(}~l{Po@Z1{&x$%onD4i(G%E zOjfwK>2E3%bL%L*6iCZ@Gp9xWp$GthHq!pngh}Rymw(^k=zi}AH$R>pv$!j1avt&t zqV=1Yv>_8{LNfqWi??I;&6LFS(?vM#*f0q(^a|*`C}esgQ){Tv1@3u$szhkdx^;E= z);%`EdMMw^SbFm-Tpcf4lfE{G*8o5k%oBxZSfw22*`m(CJ{Ib}E#wSLo}JO0l&cP5 z+%v}wewKz<5g$q4n_GOB7eIBNk#bO6>;QVjJJaFiHjvoW8s3f)%$&uhg7Z2nzSM}A zvCY}on6`h$d)tp!tNFgJJL_%ZekG#m>YEa(xCN{=gCFHnRwHgR*#=5qX{c9^silS_ zoC6Md_KD`ca=Uobj4gmLKbBmI1;)r$6g3iEdIxH_W5Hm~da zZ`$>HT^Gshd)9xyP=~A%m7O@glH;i!k$mkCa5EizVs*cuK3P3yl^|4l_Xgf3`?8%~ zS-v`JL6JR2R37|5-!_1^n55+A;(86238niPD5~HvF(JO1ZpbO6Uu39Kk#g=r#<}fm~f9A zy#F|>6%3OvDW7jMQFI_akGES85(wd~(x*eSjNuI7WSAg$6`p^OQH@cCTm%E7W~dG% z=9iw%eL-Wb9g{u5XuXvgLRLmL5B*3))m+u-LIyD>zwu6|kq8*Xlp&xPH`{td@fYso zH&vb`?{u$MXWlAI{{wpo{)UuRsc!oS=r!Bx&_V(A;v=r=Rk!=5QmVD6XFaAW&(H|M z6RJCmQUtM&)zfSA?mH@VIn*n7ZZkD(Oilz2nfB$v3ICKjjR815S#hSpBrcEMQ*?i> zCO1pS&B)&G5O*NqvZj6h5jO6pdL;dWI<(?>Ecy$owu|{8U3RZQFg@16eUrr+`_y&sCC`s`i5g}0#GC<4CBNmr(W#Y{#S$`6jbP$&jcxS4HXKMCd%`bG3rhal zIv9;!JoT{pfp2GXtiNVx`^8ANDnoH!bq&wG$<5ncLD*0s=n9jlptO4DdjQ46#OnyvXP;y@uY2n^t1|0dHPz4sUV?~8Mu$AJD^4{3T|zrqrO&%J&1kj>@kw`(#TMdi1#)d7fT!X`RWOVk1I@0NQj2hA ziXXMtqX7+XQVreTs~0r7^_Ge)?+-%2yZ9~uAC^d;Ix->Qwe`7zSAJ#oae%^Wd)^2P z;|E5q%Cs};?f&%}NV}U`%GE5T3}pEnfG8|bp#4o1S>ohOM$yX7~{Mf1v3W^s&p=H>`YflmB>u zR4iVl1!LSEto1?np$cRLS2T($Jk5UkkIB*JAqO+T(x4Y9apJ60eo6XEjpV52yU@r> z`#Akh{W`P#;s-OOJj8!+piL|YQ4VnSZG6Xu61vLn_BkHiv63+Ed>$Eo6ir<>3lzzH z`F77<8IzA}P7Ked-tcji7pLKtUY)k$UFf4)tA=LR>eH>1aeu(kmtv4trBvnNDhkh` zzXMXsek5{llS!)O#mmd*uQ4sHxU&U3L}y1ICHuXwH7nY=-h1UHRlUekA{O{t_yI4# zM2y09$y2v(SAgHwyz9@1@$j^7G=s`=&D2uYrx5lTkO9_O_1MYIi%fSzknYD7;^hYA zTGEBQuKZrIgB4K;BWrQ^{CG>A&|>E+e_+Gsxpp;+FS0{1m|u=oYmDVyG8Plc|6H0XF`~`C#8+EoUwP})I>1zZHY+#UwLQ#5A_`RyK;GX`D`~CfSeSdhi z^PJD;eU8^`=#pgbLP?gZ+A&1~!|oUkPi|@`G+Zy8|J-f{s{kG1ctO(o?bLkIc1^&X z?C&yq#kb4dj9*-<*CuKxE_YB05oYo@OjhwKt*f=~j^{Lw0~h?pHALHj?MlKcQM1UiK~CQbk8Z%WVl{ zQXLyfYpt~-E75w3N*&>L*gRb|GA_k(Wl#r7~Bdf zlcN`?%g=rsrv)N+QvJvJACBd?4Gj+wE6wXxn#C0)#-`8jUy=9R7%YmP>3284<`Och zWx}KoHr}5BHPRw;n=;y0&e7)7viGl=5{vWB*SVxEPRPFg;YVF76H1tN9M)1oTtFUn z*xZY;WnaPwG-p+OYrOK5j!v;4Zww;!>01Q!-9`1&N%Z47LG0AFHCC(0k;X~fjqmjl zD26#|jF*Oem23DY+n>eFwJyieZN#Dp2GYI_H>on$UHfx%3TFE9EdyuoYwWU4e(9(V zEnK8!pnW4Zjj{4_rf=!GC7_*0KQ?ViJ>Gy9(uOu1=vQQjHHj(}G4K5*j8^e`OAorY zoL6p7giFB_hwxvLzbXU7AR{+(rk*4cp=hZCR4SX5uLO9yN{(vK%%C!*SkZWAI+NkwKR{HdAQHs1`pr{p zmTZKL5AjfWMRv)aOMA6KZj?E<{+JYJhNY zSvc|;A&rc>%9DuAzq2ezn$ehV9KvPP?887%)lmW5^Fn;O{Es(pfa>unLuA!>e`*0g?w5mW~lh5i*sMMGin->R^Ld_Sq?U#v9yS#=5)u-gveIiEBqU@o@H32w4m=rO zJr@UlA-m})${|&c(d;83F(4_wmeunzJv71a)9as?+LsEP@I86!qy1Vop)epQqAa4U zhaN}x&_Z6JKQ8E*o)&Ul022dkGEe8VsT>|_r{6l|SJ!&SpUxuAp6kmi|9C;l9&5>h z&f_bsZhLD1hlhub5tt}J0SpY7Bvfc9A#j{g<6)U+$g(m7T9}xy;b;a-;LkFQ@-h+K z3FdIaRO+X=hKU5fpTF11pbU! zE|k^(8J0twJ^!!6TE7Vhp!58jF~;1JmF`Ha3TVjD%mf4kNDuIKOxT`a-@AA3e*gZh z#R_*O1p095j209Wl$=OTLle`}BhyX$gTS9ocleK{va&X>tfl4m^)4g=bS%Cnv#`mpHIJm_TW1>DEYw zp#88xhZjulY-pGd7{qCKaV{t?k_J6Wg#J69xxl0&Ztuip=_V7A2e6v&5@zy=2gabWGSg$nc}vFnAz z?5kvG*b%_gAR{-NZOI|@bEWp%88)O*v~Ij9~vPZOV9te0*S$zlz>iynW$3o%v1W;Z`60bb(%#$xb_sOBW3)_NolVNz& z+4?%gDx`fDjFEHrW(H8F^ZiR}r# ziQzWu>KI4GH1$1d{JT)cZ0fc7Oi=cHeivXIQ`!Oa5@J?+^D(<9mD@Vow3?yA2iUOH;oxm#-QgrA?ElP8K0dIHhp_U{eB!sgtK+K98CDbu!R zuXWj!%Oj|gf<4~s<+a=$xAVI~U%QBWP#}ahLw``t*f4^F0}#!boxa!5FzQb|=i{Jw z-;>UH^BoL)Iz2+|;Z*KQFb$UcZmy^+iq%kZYhesC)JEf3B8wX*-+(x)pdhhv1pM93 zP)droQaqoAMBY|)Au#W)a#yBpU$Dx{%cK0pb0je0n)S+c%Cri4N#6>#j*sO?xZa#u z&6Mk6M3eK*6~hJx2B3R+{$p>~v&Gy_yC3g0xsin27qLAt58WE=XZ7-qLUGXV{$vxs zt<)cFL~?sc&STudB)4;OzW4Ss)0^t=3q+vh_nkhCRKs^C-SNUsOWOilt3aN#L*IBTVLW$DYE>DlUn{`tfVh`Z?srkw)y!O#IKkgRp!@?eR-i5OmKb z9`qmz7)cHub)Pp0#|I$@w6(PloLQz7>AcZqI9|2xW;&ey!l2(KXqJLGue1<)u6eU( z3fTpXgC30uLCYJ}@`71>YqtT&sHZD#<2xBnk!bgT0|v8Ea$kRrrFwNbsYqX7J(^h| z!K@eu0bDT(=r{cd5I8l4G%{tSAK+WrZLPMep6g#1nw*wFCo*4zMD?Zrjm&ICz+V~D z`yTQuw2<(`^@ZaP0?t+iX0NMPkCRQx1K(oRbeu&od18|fkem4tcKTfEbh`y^W+Tpq zd2JVJ&BxqUs$i5B=9&md22L=Wqvv?8)PaGg%Xs-3 z*t=?5Br3gA^}_J(Ya5IH?bg$EKN=z879@Tq|EBaKWUEJ8xUsD|Yxr%E9h zFj51%Jt5^&Z$+wl^f6*h$-u>W8?M6c+k?tF=XGg?Qx@pEdg~nm&@$sq-T266PycRZ zO_xsiXpV%Z{to(cnz9C{f!Wu$c90o_@^5y{+%M7a?weiej*BT(3XxCQy+N7ct_=He zl!9-LM7!>eB`!}irEkzg*1U?Q&KC2ezupFX+!Flpak5Y$YNEBQRUUzbg=N@ipOQ)5 zG~ed#kj!JU1NFdnHa^B*{2Hi+d_u=16j_F zn%8$bXo0oq6u8q0Zf1R$H}%D9UiHWl5LC+$P#_cvh0IjE<$C*25KcE#z85R9CUdmZ z=+T`ec%e+&rbAr%rpx#GlR^v2n+IFD zqb#&TwJr9HdRpv1K7a}PO{PyY?4G>SrJ92ggv`}t(V2dRhUuIlZo6uY%p2WC4Ymab z5@1Bjdr8AMjQV2&&_7qpEl+Y!vq>Nj*CdF@wueT-?LPZdnu*${z@%Xgy6VdhKdpvc z4j1?p?7Q!;vZhGv9!nCk%Vm?&%Ni4Vln0s$?b;fer-g+4AN-Ozw3$jV|81Smc-f&8 zMuwUStvHDiUqw>!*9*GJzHk1Nv*eDroHsk@2n-ofP!@Euy>yk`EVvrx?fx-vaoUF; z{95z#>~{uJsg+sDyTf`hHhvxkA!=7xy*kO*vp@;bV_~H;NWyMT`*9l%Fkbnp_l=)r zKd`{a(L8O=p^@XEaH+rwg@Xo87zR4h{5+1_2boX4_cz1u3_DLiSzT4-r8gMF~>fj+3H7I&sE@gD67RoZTAKuaUTP30sfta7A$ zuSPBhT(v*=vMAEAxxpB2v{A<$kQrJX@LUwg*n=9=_~eQ_i9%SeJ*?caRuX?jCyF_U~`Z6)(rh z{!+?2Et)pCA^X54l8bge>>LNT6BtC~?V-_>wo`TAZ}N~6F2P$rP6&DMXsu){&?WC( zyF=kkpH`#lJ`P$Oue2TzQR0t!m#Lm7bA`9x5OzD!Uu+z!Vgs>+^;Y`RGC9PoU5#Og zY}V$}Ci&I4NruwniJv~>E+;G;9hEftE)K4&!{SXBG_@_Wx$vp$#@b4U5jKqwwERur z&R1dg{iULl|G`Z8K|r!K-g2Kw8=1$yId$5N#E!p{=Y#bA9Ep3sn8|U~|5^{BnkkZZ z|2;FTF_oH?cG}BynJoCaN(wE_C<6}6gN-GRZv|xIb)xmKWn$2Ya;@~STm*e+H1Vj$ z75JjVP=Olcsk?$Tx!os7G)nWj!}espBUs2tid>XuB-16W--gEfvKAM%@qDQ1tEbEL z&b(Q~dfMN+7AG?T|E;LNG98_~G);YdVQc3hnI(Uf4kL)!OEx5hA{4(8*!hNLJ-?tM zvU1*gc3zfAW7#0+`lq&iHh0lVTF7S+8Kx3pO_XUN@N!l=mqE2&li$)W<(USj^MF)S z*)QwgBO|N{kDq%XIrwha-`%>(7R=*Gnc0q)T8~8Q9g?fF3Tf}x!Rj@LxSZ~3xH?Un zoY$)lVaD6Wtl+spRBJjR-Sg5xLj2@*&y#9np4Rk<;fQw5xNpsV>)<=5Mx@V-TZ^db z@{%OGy}v@^>0~591vlZDpw2@0%R!ZFySu$s1ye|kxAP=(w8+nYb5)hKZWx|s+W#9Y zx){Lr>x{QvW{h9NE>-f4XW^BpxUWAj_`N~UAV(&Xw>M;UNqT+n%(15OsZJQNo3^Gr zCJ-|7v2QoVEUS`W0zbr3Nxw3F$V1bpp(bK-x*$15`zs4+dnFhrRdF$}S(q;+FHl6L zcC9h0pu-DG+OSjI6Xm$jW0L*p4+O8yLNBFJg%}*{Mpqse$j}*28bL#H;I@oC{8?CAB&ZNJ7^szK`DZu&TR;x+rf@&Fjp8za3PT<1!oA7e*=>%t zlBm1h)h;gyEzH-00l;%_b#^rH?$C=V*G9K8~9kn`W(r3 zZURZ%lACZ5Qd-vO(=Y4A;HSzA53oJt@YMLnZ%J1l^gxwJ#xkAg&Duyw8rhuxo zt{C3_h-P;;zH+r@vT3WeCP*GBc}k_65)?4_H2p{oYb;ld!^m?`T(`(B}{CYPqFk58NU`5&&=!KKS-x1+KD^M{cNcgZ3+|DFP7M~Z*v$b-H)1QEGk!fMg8S8Z;GLX7ISrq<0Mv7al z@i&so{oVC={R?!l<)2p3*!lwvvAAlf1}I*d*9HgIJpawg@-k#xDWs!Jq1Q7xo2iEJ zeFi*Q0rkpmnzBm_2I3-iB(6)2!s+>KB$*T#jod|1m{w;*(VrOI6Nrj2qi#RYW0U@& zo)Cl@#cwTb;z!vR#_vo~6Lu{{ag*393W{MEqAen08ACNAFQ38y$~Rx6FU~|DRG-Dt zy=e&+z|$)y#ad(_+G-XOz;MT*VhaS8v;SHpF!>(7>#VfAWE^ncEX@ zrlQp>Doz9&wa6zu94!39SLR_aGBYWHGQ>P6A4GX|@jaPw8xJEUDAzK+ zYwD5?^QVHyF|PvKRn)xBO@AIcnx4oY61^nGZs|yuzGA+2L^soevWAMjX zsq+tCEA^{%wAC_r)6ze8IQGrS$6eYoezz-x#z%mGL0x_8*=6-0mBP9xx(>9I1{D}Apq&%D%Sg?FuWL_+RKe)WFK93l)cwPPS>soSK z+L7DEw$>q)mLEy)O=y2FU6T_#lFpB+CNo`YF5h^1njHk^v+jHR4>nLrwM}_B zx9*T#f?>`=_}?ruUPVD;;L1^sS(*I!aW5L?b=~G8Rwsv5p=^QYh`>y-Kj~DBM-(&} zK4n&Jo0~zuY&cj5ck8W)DA)uhBf2&7r0p?^F6~xYTn~OSU}eemT4%ptbJ{ErASbY9 ztFOvxN6nmSkauN^F|T+HEwZ~Y_M@zilHiO}gopvVXpH}JYY2h=8stPE{W+aWDBR#= z^mR&NZuB4s$j7ZRDPGy@b;;MWgNaUnGBf}8knjr}bqQlNTRMwGd>}!s$|y(6^WM!c zR(^fqb;&o%tek{4q7YufW`7xTsN(t>F=DSkJfW*8mEzE0jB8u%1K$jHi^wZy5JUH6 zBr~o}kBy<~vi=N}gM@}s)(O=O#-<-FH)|HV&x9O(#^|FBPfb{jxE{&-2SlZwLu{CD zD!kRK%!ubS3fDMIx4!OV#-APK;=tT?_cJV)8El_<;a6~#n>XGe6*x&8tvq*=T{Y3r zyHcyBH?}-87Lm7)XSZ9o?)sMzTxGJfY-epDCHWovw;(+Eew7gBf4g6LKoiK~z$N54 z8Pj(WlR4n5i4daYPsSQG%e2-=(}{(bFHXN!&^0iHfWQbZzP&1D*Dqtxz?x<=y%#`S zwZ3CeGlX+7W}!OPZ&DSdqTLd-d@c{V$#J==_yST6E}}g{Fd^`_{@; z@0YPenZdD^N94JPh&+r^7UMhA$>d~`l-f?9p8PvonI;c?ghuoB44XG?kKE+5X;|!y z^%26Rha;6I>`J>g!>DIkgYJ8KDGXMgxy&b!!=^|21hLt)o5^!ukg&WhUb`NmAHst9 zejp2lkIEy|AaXTyQS3x@w$KwxJmbfnOuh+TMGq*=Os+%C1iQhA}fcjIn$xFRN zEu#=KdH4C!a`~Tmpg=ZQ^^1Eij_>JBpu)YE z?4TfvUD@m6?__I-e~aOyB&5!kWw@)B2H*aqW$Sr3q}y(qO)XbaEX_{!JYq@+d=-tu zgg^9ZykvXZvN3wX5g`=pYHohlw5DksbFoW$)nl;s`7N;YABIBUqq2zQI7{fn;-eZU zmjT=*Ez~@*J>BYN`3pG$Vq>4ng%tkVKT1>a2-xP{mW9hwKYw zaaAr3?Kp*()<|f43FFfJE9jn^WXrCs&|`_Y9D?2-;o5xC;1z(bW>O3s*d?WW#Qtxr zg|G4+;vMCK1OB?KqrFj8AAj@t)}Pq836p@ZV!Y`0?0nF1{ET5MYJLKX-4uX^hTg7D zKI2I(jPZsj?T0G*$5W{W{@__ICV7slIXK#2&ny#sgbGm!= z0SFaU&@?@A9o6N1iAttq8DY|l<0!!xx=dn;hk9XC$~T~yEh zV(s%r)_?7nU@_3gpcOv5+dQ$l6&@~l~ogzKc(UaD4s zcvrH`%?v>RD%>-I102(OawN9w;Sp-~?H#SOv)3_yCX}+?S82VFH>3{7*CU zr)KC+3!b0M31HwosQ;xEdXxWs=mh}!hXD+LGevSC&^wiK9RTB(d_tcwWJqJzQS()R z^o-#H1rW8vWM5GIe`92B8lF*_yqJ}~Uj`tgbTV=kBmh+*$Rq=`xzH^+7~x`!qr5v( z$wT>>Q92fFK^mJE$&(($%g88OJpPf4nOA zA@K9`w`^S>e&X!9OMSf+N=!`rd{%S270vLG2n|~TW#+i3sOT+t2M+ftzPxi-Qek|M zT=THM7P{u!ycMO<%LSCq) z7T)uDW7k{rTNOsQ0f|b^M@z#A7 zsN!v97yyh@&Ii!Xus}|JDFY-J``nLr!K>|_mGr(?@i7XP-=-pXbf^gV`w3`>!Zs2x z(LzHQS%heiWq;8f=IC)_8j=T&TnJfyXJuxVBBk8_qok?+=B$6Z<;$@sPade?;S<*i zpm9?@! z>CX^{P6cOSbgHb9ZOE@GRd-;zWzckZ^7{Jg2VJGj>4`Ltbssg=yRuO=06^L0sBs>8 zP+7y19DoI*7CHigV_yuVU~*WEa_IXit)D8yBwR)~jOO3a8(C(b_%aDhV@Y3Jo48Mk zy$_9|?qA|7n3$MQfILL;`gm2l!B*R|qH$YL4)H9z$^GBli{5!Pv7HKGx$*?v4 zuWXiRHghGUtR?Z&97p^U(Zx%kB-%jfJNFRf#(rJ7YlqUd6(>PJOuQ9+>{YDJ2qrf# zS>6GF=+Z+tGIAh?StV(ONGH~}x_*-%%27CgfeZrCua?vvH5rlh>{pz`TELo50%s!{ zI(lY#R@j9?gc*AbK>ejlc=D8J<~_e%7)+~<(p# z`XDGE0BKmW4fkm)UN9%*W`?-?$6$;jd2LAe!Ci*4z#(wj5q}R4;elA{aI4`|MoEF^ z>YotwQK=ObkLuqNfiG?V49>a`{jz}?#?N|^8xiNVm7gC;pv_t0bkI0j2{JDIA6m^0 zMym+4UZ#)k>U`w5PkbpK2<7;AWt?U?a<=Kxg*yDRZO>sj%ANRQ{-nNkIkk&hAo8fd1KYHJ^zmItsyy=V~FXG+)HVJ zDILBqf3=L=Mqj1&=|-$u4@tu}P=CB{#8yzzmmSWclXtQ8$|Y7Q93LB#^Ux==U1>m7<+0X*pnIkAzo{)m#dxO8hHgvJb@L`&I zX!j9V=*rr0{RBBp#QEO%U{>5zW9!+sGE+ZPy5VdK68WOR`U6MSgT!a2@7-;U)h1d|G`$r{>Xn)<;M^r{`f z(i}7YC8SRrCuVd^-)v`|qF3^HG$j=*>0(+Zk$^5aEO6Dn?Y3IU`)TQsQBKrXgHC@_ zgoG5VFSupGXD)SQjQ+?aWychU+H2f&#FYIiW@5CTv)8tWa z3H`0Zrmm8bc$g&lNUtx;Y=R67+n$b$YNpex=6{5ww54)&998S;P@8t&`SD+IDWJkn zt8HG#t$W7eFFhL9pK+UUM1lkbb)EiDyl?&0=viQ~>e1#pb9iTG_d==54|y~2g;%}- z^Xj;&>{D;?QHr|X1(&54MnuGDL+B|;j2U`OFPHB`zj|?f0*X-5)#)EP9FdPee*6od zHnp03aE#si?s-P>B0SEnOFQ;Y0LUe{;fspcpw{76VfSN+`6ZFK_=DRDaB zE4!e{5}u`vmBpW#Z!LZg-+%JyRoPcF#D|8#iHV3#A;g*tqA$^pPQ19+vs!OD3*-FM2$K(F28y4@&&MxPy>Ab zRSRU(r{hV+{LA{VkK?gv?1Rh1RA_?zFCq0a7ETvsU?OW&`9)PVTG07+2njay5xx~) z=XdkD;qL>7ie8x!gQS!);&9;y-u}T(nVB>uX|x#oc~az}VLOPp$^ng0ddT|*1<#a@ z3=M&c+zA0Wt$Ac?#w%Fr{ZNHywlqE5NIiEnErVV+J1x(VVBSi>@?4sE;scT%R{F zekyQsP}t8`o8*@?JFmw@60>s-qD#T8EG;({ZSoZ2>7>+(a>y`|*ztXX;ftZ_s%)F5 zO~qaSIx-2cuA0x44OYSGh%{cafdoc*0zxCA`L%~A9+_;?Ga(qgcv37tc8eaq8N6kT za801z{mjYd?;!s3VG|5_gO=}F%s>(?+il+pjBHJzxWaJ~>RJAMu5W{Gj*=Z3PI%*L`7lq;@}PGm8!5nK!)_^#PHX z%ijq`h2KrRb*{)zXb;Q|;*Ws4`duyB*z<32YqfFYNqXyit+x6K70nRIyk`)rf1bK& zQN9RoU5KRz(d~`p=t^Ki|MWklo(7lVOb<`;Dxffs2eMneb+I=*h9aw9bOZO+=SjVi z;2GKX@sWMntvU)ZRIi3g<*$vPrKxa0qEsY%*REYR_9`s!5D!}F`~W?QQ|@pY{jjrl zq6A(~h|?R4XXM~Ry5vGDw8wJGxlP1WmTX@0aj0f=mYc=uS@nuCM09KH_iII!~=k0K?%RlnZW2>g##6T08u~+Id-Fukuo4n z%#=_kWsypJ>*dul1>cuC|IHk~P4Wmx#P|W|TH%YamaPN(;$)u!F%ctk2OtOl#0!Y} zsgPDa+g=lN^%!8~Kd1YrZ0*P?oIGD++lwnU>jodV(jDl?+dx0@U<*xa742yRXFzYS zjw(kaGPVBg1?6)~I(OeU37D*UEIWx=9Ri?{u>DmN4)>pv_LDB%_Z)*x|7QCXqWt&# zCM19L$KVk7qnM0nI?Q-Z{PyWkM}#+V_wTYMoo)o<7pi9ozj@wY+=fVn?~f)0@)^%R zx~Czz>Mu}3@-m(wTDgdAcuka{+UCf%X6y?EW{>^Jj*TBqYehg<&#`5eGk04~cdE+= zbKzz_`>Z=`ih?*hfK%((pbDwKe)ERYS6ZEJ25!d#VDNrnCDe&8Lra+Lcjt|_yatt$ z{;YNS(#lmGNgsl?z3hr?k_|H>S|)5IaCw9P35i}i84vbKRRVkweYx;W^^t}luljpQ zhGPJWejlN<0QicOxX0n&1o*yKKmWG|S}bzDW0J#FvDRP>+W@~x}fX6@WmCI)<-qIZ!nbXfhUJBEPk=|u`e^AX|E z7%bb~+4+c7EG>gp7}>QfN~AL5i>z+-SE<;jGyN8TBtBP2`aejT`rlt>=OB?cl5xM^&epEY?;Lm>5gacIc2Qg^@z6T_ z4N#Z;lSPWN_b8cBqQb{&KaQPtoS=KzBUz#}Fa?X#lFKlM1!6{lKs_$rcJ9QVxXd5IOrcAbwW4=nLWfvo4#A_M8kAUPBZ zT8b5hrU0D_Y4Tj^xqOmXWoj1|6{WmTAYR(EBPzsy=IurB2g7AEGRsEY3}mJ$03a}zA zN9$3HHr<4p7qjNPHb%u;Dkqe3>C}kX)U>Pxj*))6>|%K6-izyt{OW%{DHOx!*w}%n{*Uczeppo@b!4|>ZFxOwy~h> z{7Ku&g)F*0Clh+Iz94nxh8zjO%LKTH<3hgUIsh~oe9FEc$r7-Nnu6H%l~(1xr#9jB zelEkE_qw+p?O1eC155=16&bOQij&w+eEHSD4t|jKy>fI%&0%9{i$0Ud4xen8Uk8qu z1Ik<}zsp&Jp%wY6-20v09yhEbm$Y_93ym5Y9+nlnNtQSG%$bQ^{yq9zdtxZ$IGG-{ zxf>|`!6p`c9kco(*YF@XWpg;p7aq5eyV!D;!)^Nx@Z3*%o`(yeJi#Dt4rv3)6K<$Y zUv!g+_ls!%y|k21Io@{r=S`LyI#F(J@kblB$K|>_fXAZy%~Iu)>IAKzS!}Az{ezikNV`Du{%p| zFZPSZ@M&7z61VrQUQ>fY3s_oEH&!t#qj@79N9uA6#kOekZqM&nVdpr>ganZepQ!EN zzWeIX_0KFg?V6&(*Au+m?_UfK+&C%L*Pgd*?PVhq(bGlX-4e0<5q4o=VPJw;N%zt z9P@JA!|u4?X4niu4$1S|e@Uj{qmo1Gy7w0cCIKz|>fRH8%<-LCtHDGnFg(s0SuTqL})!30G_|GbL->+%J3=KRXdybClmZ z+zwe^F;kxl+c2z36`Q>soETH*#3(ZM3kdP%`HjOoo=Np&8~NSEQ9UO97}^Q-jT|Bl zF81xBuX^?;d4g=G5AH2jtpHV$xLCOmdic9=tYcf8e++A=;_ zaLO+8v67f<7Otx;TO%$C*dqkLT~3&GsCP`VBWUGUaI#sQApw=akOb*>iOV6upnw{J zQ&O0-@datzAzXD4zO}X0Dec$I`~dKu<-=lcSoA%GsDVBXtnNQ(5b4mOIevUxYZs)! zJLkLhYZawk=k_F6=tioc?EE@#fv;Hm5l0u7j&MeRuH$@!)`pQ=vo45 zlfb=Z>$vKM1|6Q^qfvDxel%+rfI%-v!ng8m#)10?+mv>0e<37?pQrQIyMCP#eAs9- z|GOkg@l6{0)4QuGpQHCx<#q|WV0ac$8_6>;|_OmHII9;o4RQhB%jBt2qX+3$j+ zGdp))rec4xYJevVjMt9ff6a**E;lHIWVPq5QO7MraCJHwKE%~2^9=s0a|$Kz`(#HK z6F46hRFE&ljmt5BWP!^p!yDe_{%?24X?Q^Zw`znVC70|@YvAgLzm|&G-oAGN3pc)p zyL~n|)tkzwewG(8}DcmK{q%bWR@KFf{Vs?2lt2~Y>oSKDc6ksgT0+N+x>o<2tILL^AEZh zMp`*-#x;qDTQz}RL=o-{P~O{Ey*GbJ942h~(=rm9bR_-6ngv|qJL6YNi5u6ngiW#7bb%=l|J99_Vl-1r7jIZ zo2xX??BCsc7w9zNewCuU)cC{nhpKAn?~Wc4X`@ab_xXV2sZRjGctYk_4Pt@R;Mcn; zFaEtV7Z9NJ+L_^@4yn5qeU3k3`LC}_<&R)ZMbR8()aE3QIVUB~MQs3CTg76I2C^oWOjD(I1_vjnzM!hg9SwW8^vRp$|t?K@rJvVy4(?EfF z;IfB&D4lh$oBiqgGonsN&CbAVVxHg&?0#tjtnc{Z5d ztPa<^DJbxkr58M(}ahk*A2_gop% z+P+UHdIFMUU}}WN?F$qe0@qwgse=5NRxoJ4ec(i5;y*^P^#PO;wVu&V0%`77n=vDp zhhg3C91lQtstWtg5)61@^)K&6ydEz&eD02#XL&Z_B#tMpe=5Cb21JaW)Z+7W01Fa6 zFVS6U-V{6&O;~_@(DW&G${^F5r`Y=A@3a@9;5F~D)%+f$m1!P5@8i?Ysk|ZYs?xh^ zjG8~STrD}A9|cLl1pkP>`eGe76>^+AQVwX`RNt1;LD~OIQSUV^e;O_y%rJ=8=Ow2B zvKDJ5pR0r)0K8CkFr|tRlHl(wViWRrEq$U2%+yp1>;a}gb($M;LSq|d;{|Oe;Y&*G zoTz^MBQy9Z8$-ip%bz&u9QP!DwjYhFPdpTQkQs*`o^w5sPQ-Fxlk5G4_HZgLFAqo; z8!hZ-)m#_4zTO$fPdduWb)An%xD$fThtF^W*qGEWh(PCuI!2`#VPaoze6xffpP_$A zgWu3@KJS%gY~wFeV!|XaeC8ux;%22E zPWV60N^d@wXScLOK^$OQrPgV&L7z==>d!7ZBDJnjTnlDxJWYL-K|Dcs0m+7r z6>x0*+5wwS0<`nIzaxcl2z99&0eC1G-V|md3`{rJwfzP>6HA%=|TUcRs{Kunh z5&OiKSc%ftBy{OhX*n3VO}WBXjMRVo#-9aQYMR$_WqJwm(gPywE1=$)ISvqUX!SJ& zIN9g0dR&lMi%Y?K+ZI9FxgV6NJSGPMFgI91dE_WGIOr;gO;~bd@6?FgXojuL;j#e?&)PZ30_iA)gM~_Zeu@&rFpCY z6N7LDh^!IjQqh`7=mp!e9%A;kRccwm}-zjB#_yGL@ zUa-Jp6rl7NVBeGz-gO-5F_{n9F5g@R-S1Va36;~Zc)N67Jg5jhJSmscc6B2_&iVEA z$YI80Po(P`y|`IS3yrEj9o_}D8_zbvYzhh%76G`8cERY7V|v> zXyt_gJUZEUX)0yf`!4gT1A9P7lNkJ+KvW2$Mk&&J(pjJbeR4*M@TmZ<*ajyiKecjx z>Lyucr0m`R)O5^i8D{Wqz=uLQ91F6sy@^;)k}=AK*Vli`_!f83p!c1{tam4R zZ?e(d`*qkeMlLSm5zJ=~3rZdysY5*^u&9>vo5w_lnl^EfVc!v_xJ6TRF&nXc5qbAH8tAGz~f4qx;G}XD5o`+*55bJ ziqBh`w3S~ghIu2s{ZEUO%_t*R`hs0ZNU!*pC8tKFSJ~0I=R;1OL}K}Q6EnXQ**EPv zOw5;FiAA(r{=EPZ#}5F~DO^j-fP{Hi%hYz2_vyQe|1{NGW0JRrqX2p}!{`<2V&?86 zZ!RUJUCWovm*TIug3~zwU+u7-3Sn@0h63CyyI^EwjEIQPveZ=1Yt1Y_IzK@e1JW|y z_}R$31WTLAFG+#8R%gmL*X(#Rhd@a=;LJt2U*3}Lg)Y+gBE03V3x59p=&$I|Y4F-T?jWGTRAM5n`1%D+@z6v8U>zr z*c?Q}n@kou#!m|>7XYbVgqLqRfT;E*exB4j@n4d-pd$j|DRX%_cyvpNe`gDL9-ZB4 z9UNp;H8o|Si%Eay3k0GDuwmq>|9A97C_qR`T+2VmA*gPq zgB9;fX?K^rFKTHw19~HO((vWmO>HxTYuNWB z8?tB*ou}?Xn?!TRT_J`2!%UI_OPkMl4SX5Fh-#+Y?+8E!8e}(JQt*GPphcRs^L3iayrlHyYY5^kN zX>Hl0DL*_K9}nqS)A5?63sYE?YY=eIU?R3||3sU}hAmvZngK!w*ngx9MlJao_#*+@ z%Qzqo{UYXoIP>M}1?Gz)UPk{7I**BDcyy!;0&Rm`L3adNMjYz6xMr zHl)yW5|a|Xh!HBM5xE|~8ChsR5g{KjTz6+Cvo%5#NflDUXtnB0eN*Tr?U?|7in$?% zv73npf2DrtDR3H2wjL+%FimpMDm4eL&YqYVnFX7nmXF~R0?LxXKOZq_zixeYT(6}8 z;u0p{p#9=UT8*ykfZe*0GGGdFIsF+P4u7Z}z8540lp3-g9%N9}WQDC%TJ2I->JUS% zc7Cs@VsTj{yYfE^O`E)q#+?Lv8uW@X9|-J)yD8sNdkVkgt!6hT1B9d|mBjwga@2(u z7bc!usW2v3vCG%%cCEViQo-7Z2WZHTvKu|e>V~2Y0zBCg#g31$)!pR$Fe`N_8Q^w& z>0kvUFC`-|r_x`$Bqx94LxcS~s`~PpE!I*0#*1X&^)z5eBRxwX|G^5@ZYBu{>Q|ZW zikFpbWjO(q3K8oY@9LOfc@nMo{^K{VX;)IK3{g#d@ZYEz^9O(X_Vjhd4Mav#`u&zy zwqzUq<|NE^$(ti8`G;d$AfVfmWYuSW_}Fgnu`h_1kcKGALHts-Z+D(LCIVP{5+1ZN zn6sf6zW?YWVaB@*TL33gF|c#wBmj0@?sG0eP^Y^v?+rSkHviwR&HT~fk6x8NRDm?OtQl}7AbnpGJG~J{P(*Dur8Y-f-u2^rJwYsverJFw<~wJ zLwaS_r+)#T0RA5hoWQ=j=B>e+^*3IYFRPF5Q7A~p3;-|XJF{ke7>N;jnRO+=!_Kxb zV1!ZRA%9LIL@D_UaQYAL(Y&TTCx0kMaGN4K`AFfR`=5sb!6&BeMI=aqx)SV5UkV7w zr<+V%)JvDY#Acdem1>;s5oqK|qL}JD$rJ-uFiB#0i-Sm0wSHl9iQ+j3_z%2x{6EFL zbyQVR`!ySTyWLolhh$A32J-WPTtA|43AO-{CDB z-kl=RmPQtGtb1Pqgba#8QIHs%QU)+lMrav^=tt)?(BUyAGMGXm1Qc~&(yvml)(Ro$T z;lK^Zj;+Y?ARd)7!k~l!TiX>$+Y=tBmOP)*wq&qJX^kdkw*WcF?e%@*8P}5liUcqK zJ0w=GR($%tY1i7`DIknhX?uj2ES_Ge4&?}_n4MaVRdmeQg0nAyWi7@7piouqhiIv^ zIwCC!oF~1O9`IFN@wwz2p(XACxl>FsG8=YE5J<}Wn5!;(kD~scrIU@fT`DiBmTQJK zs@YbDN=-hk9BD}4GHl{jfngdv$K2iVp$n0C=?A>mk%!1igihbfgWfteLuv44>;O}Q zATs!ItXm^$e%A@IX}?BhS-5($x92X1xW1ar0(F}pP%oPyy=W@{ZZJeJgLFS=8tX~S z(-`u0SBpgsSJQT?#a8F1c(9wKn!U+(w#RKjFGXx@tZ#~;H{tXiulQB_uhVa-_4iYC zj*i5~(RNW=Y5?TxV$bk{juJ10C2_5&XxHg%el`2_%K&!UFk59QlI?ja3)p?_ftbij zF^}V$1TH&6QVPFo3B4`OeBt+B#h#VlTdTXvqvhvO*?1u2F?kVAZ0;P;087c&J3u;5 zeGWE|i7)%|XdxKr$X`VUf~%myOZcvQ2PfnY(BIf+h=D0hMM}g`y9&);HRbzgDX7MA zAbeN~Pd<|0UmJ@0t-e{kr(1CmY(5h8zzs8xHW?x~`pl8#EQ5v$?;YIP9fn;ENTSb4 zPrk~zZ^*c2Tut(kNxWBoJ%8afu>NmLQC%7K@N?%EBfNRQ$Aj^;A~;29y6z(!N|4;Z zL4_0U9!37BL>7~kp#^Eaw;|`OclDchysX0L;0n}(EzWdzSPj41H@IbTKPQW3}3z7c@1=B+7YeJ+lQ7AAHpWZ+PZI};9ezQHMz6YbRA z1`qdMiwN|()yURF=iesV!+R^AQ8G|Jc92%6&nX=n1U9F(?YScTKW>KQV>y8|^l)Q0 z^=$@T--f8Hd@Ju{&tDepDZDk+D%!~d=e02LY}-?Plo;_hMzITD1U-~PR|v&ey$|If zGkOAqE;%Px7oiv(q=^NQkIq6B7!R;C!w+}7n*reH?~EOJ~=dhj*)_3jtq&i7`}0sn!Ql8VY{%`{nz8WZ`h8K8tG zYawK1*59>jk%8HX6rdwj17Dv?7pK8QhS?gkX9ga_wb}FrWT|goG)uuEs*W3M#_GAQlKF7TLPjcTW;8L_6Nl`}?QsO4U6oK*$2n}jwSJ^U)9cnV zD`p86hMoX#6)}5z&iO}kmd#jA_@0X@n zR7&CUjOC42dPQxi6oiA+*L_y~P}se09M#LzpK^U1dIQv?m8t8~Wdi3Mz{X3q5OOP_~{uC4a!jjESvl6?_e-sPs| zB0Hs1L$4)q+L@4CdT&twnQ-@!u$8O_1;DfdFf4%w$g!lY?J=$yHa7M&>sOgLN_qo5 z!!hRh1vUWX?M6jKISHl!WaH3Rj-}*2>=?Y3_vw=eVz+&CI)y@N3PXJpU6OvS246Ia4kjK zjcu@AuXhU$xS)gh>{s_C5*C9Db?R$3`}X$A>EnGQFlHi!^Pg3vMW_Mh zkj}R2pMO?KueZ>WJr5_9L^aIuxx4ICzAoDTW`P3X<@1L_*K#omZy@Jd;gRmHjKk$I zlza1roN?w;E}n1Mj|#EaDCa$;?iC;sCKj+qH0h003EYkL1mcB3nD{-hW&=JUJ3D(a1ZY#rJJilZ>>z=Yzm}4iw91fTpAn5d0mR6$A_XGb zkP~1%LJsVBSa2fnSO&GvWmX?qJ3xC8lFnlV0px6rz(Z9vd+Z;Lx(>PZ8YjUvOvM~EUpmIwTu|Wz_*Oj)&Cwz4D5#U) zU21PRy@UaR-0m~OhA)QQc$KSz`PNBG)u-l}qD?bvwR%suPiO8)Ug7zrU!><7Oghu6 zwWqJ7y35E>_e4fMcM2B&>7cQlCRUR_w*{C~;vp|d?8|aL%B%iW^qZ&qp^Wy!6N@}7 zCIVp;kNX+{;=J&b#(C&{9_jdwL zn`XOqDDqLzIrBsBM&Gf|y^`S$RvMTPKDZMfkQ7Sgu}aoh`O0+j9kZEbK2RRHnP!V~6Is8Cy13^3uF=5jvQ&-{jv4rY)|7{IjAz|i$RI20$&tqOk zpq^y-LV=*E{BuKgJc*EV$*-TMKjz6Pi&Bfz2TD^?>f^DnPe1ozuAv|!j z(qy^D#UIJ8dVMlU-H8C+!en_x^Pt(jth2}EzDnRMpbRo)ivrrzr#-qfRq#i%X9`=?%=W&qc%*EuK1fFr#BD+HQ=sJ<H6Ewtz-`3w;&gxNf{=D#=+5@kmcK{GU%n{L4ytSDXl3jI$zgzq@rR*KT`=p!<|}!2V_ryo6$HWSRW*N6FUA#L-!d2h)$fV_$PPG^ zx D1?7|G^QZrrS2OEFTB7?q-UV5fOy}}gPTQ=F)%N9t=B%N)zL)LrBpnk>pv!iJ z*JE}ylFg$Nm^>UN$!Kec&`gm6Nn&wLEEfYEo}BT}mHNn!yI+SvvhK2}S+@4su&^7Z zr~*gb+}pH40F5>sU_1HN(hBc9dfwL zqnkCXtZGMNpganhACu^EJWs<#U_k3b0BoVwx<@(2UyIki#7_ohbtpaIo4gv9I<2G- z7hL)Dfk3FAf)%xCz#8>V6k410?)6NJ=<706;|ML)lqNK*Z>009va{!{ZktJ3Qxt_% z6a#UQZZekTn3EydfAJ3yv%M|{C3w@dqQ1N%W+gc9q@b8AFkwq)$s%Q}Y`-bH;THui zOd;t`1|cmvUQ$*ty+(dkv{9yQV@oDIjO{YnH4=nFMEdTC#<`HswkYzCYk`ezo%H-M z?+Mv#D=J6NU@1mLpK~Sa`P+Gd)TkoTY!~HQy6c7I(850>9b82(ToBKXYiQ~Ymw}#kz$>r_vw>AgmNtsN+{)@eg)`; zAAF$l#fa`NPelC2yGKc$qn9yo({UWe(C%X|tP{|~I)bC{%0pPI?QC)XJz`93 z4IkqL@5w^6tbg5IK(JIVjE@bF&O*FbxrbvT-F)RIinsc|G%O|W{mA{2)*!u8`b!^f zAU5UacE!Nh)7RWZM{OVfP&_@Gz&lv4Q=2@>#&2|u{oNCUlJB@)AXH3yZ2@BzucZF- z%iJyYcaSe@z9ZeHYa=S50_<}NEor>S5%uvsT%Y(Pe&6pPKi03XER<$2@2LM;Yy1)5 zNJ{O1=>?|3Y4ZwWhHk(Qc@cdnLj>Gz{H8bMb`X@voPrsuoGc2NXn&{$lddWlN^^er ziRcuNh*}Hek4JB>N5Ll;y-~A$S(M;P%|bF{ALd~du*E%1LCF% zB?k^)^BDTEyGJoz?6a+WP-uK(Uo6f>l&yuqhy2r|5k(>N=~3U<#pEB4u$&fuAkW6= z_-*J|`x?N!$!XAHvbx(zz8tPbC`*Socw@4pSyx;Wa9unp@4n>BTc8yVpXOl)8hxU6C#a0dVt{)}6`tI!Da;TJJa@)J<3KmQU z$?9{o_YSKoL7qM!Ra7)HI;REXQ z2|i7iX#gCC9V{K2j)>sMO2KIkSB@#Fk}2vySr2Q&0eTc3AdN9sG%D2c_wB)$ZFS>% zRFzVm(2Vx&DdbHXoHqI?L=l}OaC)z0LUjTO5ezjuux^X{Gc zz`B~+iH!`^w46v5=5#pghxA7z?rx z^oT%I7YrsVIWv7@)A@F2GJK58K?& z`e?Zsxwgm){=l|DF~1%h!@cmfn;ccxz}Rw+(&C37rrAq{b@|dAX|pmT>@USqp{1Q8@h#%tO8@}|UTBfJ#@EZ3ql+&sfKzn|I_Dn9- z_Ob}&dClB6|CWDy_UIDQE-*e#Md7W_E=xgzYHIe;cyT)g^eNp;*%Vw(zk)5};?W+>kzYFy!6hK?ME?!FJXx0_B~R>-t0U!nia9GR9OB3>DiN_y?;CR>>uk}*LD!zKfzAu#a%*U) z&<%FuqTQIi=3&c8ztG9u`y#He?6H$=0B(IjX;xz|2BXcu5ZzJKu|(_bxWM3?m~^m! z3Ezl@1E}=oe~vZ0?aTe0^=gN+LE`VCuQ^|#zrKcpM?hd1^VrEFeY{?-=}#~h(`Mp` z?o4zE>mHvKa0ui#l{wMrfx#w9nDsp1-vlN6p9v~=0U$Nd{R8RPct_cb`jyo%w?%Ac7(pk<#TGVsKSEK4AOUbua&Q9rFaBVoGyZ z?4LE>1dDufcG8MC*J&=r2U)8&nY3@27##z&fP2jamXz`Jku|KmF7zz`tmY-`#sJ3s zHRa`@UQ9p>-#Tm#ky>A_ds99PFN9{g!RK|o)?|pS0 zI1P5x30=V;Xfq5G#e7W7C*N_r(7iW^f7TCdTDp6BDERqjZ=c9V`1_(`uEO00J;R?; zvb3~RR08R*Xd?IL5}si-NV{1WK7OlbOTPf)!K3~z3aoqNGy<}5o*<<#-e_l{UP>7z z;(6sP`}?EVDnPOx6E%M-^do?sba}tD632du-;|xzUYzH=sdU*jaiLVULwJQ z%)CfyLeoitq8}^-=d^nh0c5WCQOAvk_mS18kk8*Y2qRU{i4hRm!vEK(_h9X=Bw}>H z1+KwLo<4}IhOa~7sHFW3gdTTc%It`b@Ff61FkD`GXf(7GWB7EjU(HQwQs49Z{<89iaQoU%fF z`jmsLWP<8#XCA620D%WLYae1$RPx?Q7Uu(!=YO|VN~A)*M|~ZUFG&PflXa~*=t?G- z9>4Kc`o-4m`); zRwZPyG~e}V$%Q4n(NPzJoezEq5?F}{DH);|6z*x(1d~{V(WVyTxpl@ zKZB7Ba~&#gm5c3k^?4#ds)81zDjFZ{GBG+9*kZC#g1}@#Z3eb$U`Oz{$LFi4!h8UU zu&}y1?#GQT@VFnqUI0ZBJ^*nK>htH_t6c%Yf+(70o|pS`{I;vUzgSTa5y@Y_CZ>@~ zRDS9@n8dVIwFo^~@A{^GDTdAweEzrfQQa&Vf#-xa3{-$8KpI*K?KhQO{suiaCdrt@*5oIQIw_}YdxA+ixQ6c*44jKrfw@1f%O z#9lY&T|kZ+(%-*mz@SF}(B8A}0gYE6XYhy_rmn03+4Cb5{FodFxv+(E15gkP5u+YB z@)*vf3@wQP4NV+2#3eF93if;i_{|mT1#LUnoo>hi3lzqXSfE+Z^t`{`Y8nLU-x+0y!twz&36lU=aZKgZBZBH$v?ID%Bf(w7u6;FIh_g7B=XwkDt8a{^V=i6JF{V ztXA|+<(W{mmYIzXl$8yBPM9Z>F6tu!ob~hWvl0^n0lsPi{OUnzhI}2IPX;w9ApL>^ zQlyzTXTVNL0&PIW%mN_GV-I9}-3MAB>si`opI&DLs z`3A5`V)5{zc80r%l7fQ9daiz!^J%LD+sWBA;2~2{P{2{I0r;m`0>-=uFkhgjMqq1) z=IKb#0|dIOL?s{R&$)w(#Ep)4-5Hy1{ku)y`ny@A_(Oo98?vjq@e_3 zW-k!2>EYoiwK3RZiNmmh(4b!GPQCqc^!SZxqLU8dFFN2bw2gI?6#xfLBk26 zWjFw)b0n%w5DGMObbXGh#fl_NNA0%-K&9U26u!F#Fys_q%}+dSQ%m%bE`l*nK801f z@~cm8698(5Ae}25GJIBl-XT9HAP1!hxDNw%&tk@a> zVv-zKU2z*Q!A7k4r_OrZSWHP32!Ijxat$CZt8o#UIDh1>#f=-c~uvB058ezHgb;m#VzbW08<2lH$9 zk%LuZie@tcQ73zg38FaVZYanI=mQ-ueFB{lZ_fH##L~R7G@!ktaA3kAz7>=0=L<@p`ey0hEiGosy$ZZG&k%%%9gpXG0{Me-r6C2yVRLKb(1o?{3i!aCh z@}EiXS7bd*qTdMw;gCI6O2`D*p}%1puJhLRw00o>Mdp${x7${hOJsDW`+yBVyX_RJb#vn(IIEE+RYLc)3hAknC}L|E zc=YO0%s8MO#R6dxXsf4?KBZP9I{K|JWG~TT5@YOdZ+A_ewF)&2FN3~MwGtTmm_ z_{>k%LPe*tuzAqE9_xSvr&3fXd+vKOpP-Y8=z%%X#W9`vmELc2XAXyD1UhUFLqbAA z6(w#zZwsukrhlDTsclE)Jw;R^Fg7Z_ABPx|`j}KeL{h={6;9(F`Het)DU|F2NyrKh z4RIPRSWywVfW}DWYrHl;1YKj3zYjt^26i(htKDM*)Qcv@T5NDu~V9#nEu2dVYC*!mpSXD{rPjy80Y1~*j^RiD&}L+8wb_Uzc+;2 zDa^qmNuXCvT)(s$sxF3-vEezdSReY9$dSKzEuU!t2gB;n=&(h5lQU|(PHiR>Z3x+V zY63JH{v3}s2vYbjPhJxJH7Le~P@`Z*pxnTe-$r#ui^fNbsy@NDq`%oP6(7S}mScuj z3`>IU1O%b|61YToK0A)6rRzWRzV*acuE^5vailJ5zYb3BiGrSF67pCk&V}W1)n_w* z9gB#I_pa_;`XuCiQyqa^jj(^y^b(z* z+#}{v&{m8eBt)Z_80wMx#~EDdzLsCf!_NJztUw+p>d@zSwd)jAljw_rem6%SeGx;g zsIV3o6cH3RNm38*-x)9apFlGNfrh;s^NdzV?X?A|$9}cRYk6g=+~SWe(QHCOJ5o`v zdfve{!n-O(^+pI4v4-HTBwy=2?nt?!wII?-e)FhV{H&?7h$6s$V)Q6NKXLYJt>%nrS6%KH8CUnnu8QGo>bz@3I zNL@aE4^PB+!gPXi(~QdpL3tOMw;vy7-{Tr#-3n68w60GPD8qh|GcU5Ku<-X(mW2cb z7_mCu#ln#A^~edtR7R^N`|j)kLiTY>5M=TVyOm6^vb1aHI8#^J(5@o1&@19*Ndc?C z>iG&m{dnP-Tghna%oA2U`8m6UpJ@q5#Z2Xv0xFQL>(v93+CXDe+NubT&W!MWb38Sy z&NeV|`VeE>dWv!v82&xuOFU^vLyM&#%2{A=3W2SkeA;g@yA4MZzQYbWw zz6{4dmwikuu>c>e+QC~SqF%tJu|30=^L*F9{(;|+KJ>n{IGeM}Y$%5Ka*@goo>}U= z&*Epe7Nl61?$zF@+#r%F8#$dEc(bp<5^)LQ#rF_a3A-C-DF~PdCBwi`ZS1PP5vv0U zSN|BGmc!OLMwFBi0m{$bsAJH}{wAj~uOGyA>&(RBm*q$3R4fqnVA;ju)G=o3v{p_H zs(T5&$ZucY=w%2`kmLOCbZBEocmJkK;muYi!R+_^L*O!ywS_spl0Vl7xYlo?TF}RfH9lf1?bKU+nV( zuu+XcL$Z5PpGoE9b4MM07QvAw%8)3oLTpdk!wC2j?RMsE`!&A{01L0$8M|Tcu^ZMe zIfM2}=rKeGy6MfgB}3(CW(SfrpR?3K3S%IDg6oFsFKz1VWAX5HeSB9+QKfn(a|0*h zVtFmhw4_c`xIye2Kt1Q5))JxhAwo9*C5seMeLvv1XUl zkIEmGvtNj#Ly@LW?dLle#&ptmulYCz`tC1X8*At>MKQmp=-G#bwEUs<#;EFE`KHa4 zke>s4gJbuUj@lmbCtfEY z`|aw_qWAl))8bh8Q8UoKRB0~6DH3+bn?*fXLMzXAkd{`m_WHY_#e92o5CovJTM)ymJ+ zr`PW_J^3DlwOh_j)GFuY-IT6K?|#tR1@d*E4xo^ElH+WOJY4xCSpLYMIUfmUI?1mf z4-vKq%j)U9&PqwI>sH?p+mdJXLezyXd?k*Zcj6*r4TGN0E)?RT9^^%^atdvL{UO;M z%<$nRMVTdn3@;4ieSc0Nz9BCHd-8Ndi?7d~M2vw_a14M{S;P<#s_bh9JP~MvIGa2& z#YhZ#iyB*5c*O~22jAYWq5B-evb7=7E)Sgu=&6iat1i2x#iTPRyS#FI#*2ly^8AqE zV}Y|gM*IkppHKY)?QkNBW93SG4*=oi%jqH2xx5`d+PK8T2^WeS*Sz^p_^|uznpjg> zr*N!}+r8ZGqv*IwJULkVs$fMm&*6>Ag#_ne4U_5BQ;S-nLJ_tCiFp(6kVN`}ttmS- zWLvF$G+v`I=8v3&G_0%rBK;<;<42k|3|CCWndWWLYFfB zA7;uVK?E`4oX=(c+^@i^3H&?|gkE|a0GJ%3o(8oa6~yNiHPAMcmb6t#9Ljv(V9LW_ zWUAtZ)>}QPC9>oK&SlB{a~2M&aG^5_fRo_g$20P(>(ad!lGq&<*u$0xO=FtjxCCgGf-X~u z@q3I$^la>iXYk8Y2u58&bzgdw@uU?|Oz|W$yOo3#-)8!jXm|d!3{pd;6P_|KrT#Jf zClR`|437zjV3D3blp$A0S`rYKJ)}F^$0G|DA4FQ~#aBTjNRaQztvO$qE!G<(u?r@= zY~|{nF%^L~Qi>28U_{Q)47Qk2OAHeI3~+2T&IdT6E`Wa$JYT=9KL>o>-lC79b3DUI zMo+R^?@e>A+e+M^RfL2LsieRjw;!b*?iage&eO0eB6OO2;FXu2;LAdcZJE(YmeVdU zFeYlrhKGWe2zWsYMn^b~cg*LRrl2bv@{>6uP$FbX7Cp~~X@djQ@FSS>x{ogL=ucXx zHfn>gI+Qxa(seb^4P9lp;&b^LkR?EK3b9`GBWS;QwSPP=AdS7-3AudKMn!d6(BGc~O{ z?wlP2c+sY~w|BHRnA;UeM5PSz)WdVtPM9)n)1P?YJ@v2Q=Kv+Z221?s8*F?QX-c|L z*=xH;jg>(eC=;PvQak>uTxTQx$M1LGZq7IfS;YZ8Rleb(oj@+_3=Z1Z9Z1^yt$NyKf!$fD-;9t%l#Y1yvf>t>6jF#ehLz zWj{dH2%TjP6K=f#=%IBrx^GJlcSsVS8WS_tuEABo*VxjT2 zp}+nFmQI9k>G;`bioC*}k2db8-~R2K`& zxmxcfvPlH8>1Piv*y;@XopT2*O*WkE8NV&)Pq^Bb!W_y`^sYu~y(?KQ3)4ZsMOn|M?6M6bp(J$+{fQ}h z6~Oq^3B4XZ8!p%^l;Ei}437zq2(Qi7p925ZS6E(%SmtDpW6F;9-3TR+n2h7gPPWn?`@=I>5{8Cz3$Y9 zo>qNUDB7wg_60rmwJG|^nU58QWR|}UjXJyM<2mc2b!<4AKH9M0f&R>KsP0O#+vwHPCUQsAg>iIXO7`V%_hvu(0 z_-KJBd*;}86dgMXrgRZFCJ7;LtI!~sllOLLIZ*^0KRvmDe{vIWvK@Z7!xEjT%IKxR zxp*Puwukx4_zgRrkhgp^s652WnzHXcARhQ(HhhxhqN3$6-Uqf$R2eH{!*z>{n7gH#fK5=q@k*sEU#?0u+zmbrzG+0xH^9Dt{r=_Z%j@ z1X@aZawpAVg(gS~PoHt~zenKoQG>X6y_jm;dl}b(w;g@$Uw?Zitop-wztrq5>c(mh z@~Ua?@ASGx->4$#1u|wGhNuBQ0Tsv3np&v~cub)eP1P8pT`K##Y4|Ol;nQr-Oo^SZ zOZ>Rh-Rklg-&}(=N-D<7rdEYm)b(GtyDHtd2vqAZbO z@L?lQDgF7gL~cQA@5F9E7J|zXt5;GvVzqq$euG;6zW1r0S_qdJ59wqRS((;Vo;n?Kp;L>Om`V0s z2C;~$PV*%h>6xnmYo^F0YpFWN1%VAc+e}ZnLfpd1x}R@`4(RQ|yUmWGTZID8>mzCE zqc_RVU){78f1r!@(yRZXV~!S9LJuGQ28Kzl8PFKIKH1($FQ^9S8rrC+LqrcR8~9}9dk&v<{KA8`=f;mSUgs|>l(4q!|LD&+=# z6isGJ#@eSlzk|=k)e;bSKgslg&L2=y44=L93Xx5E73VOB0(W}&WzX!;BSv-|@5{zS zxs3w}WTB^avv!~|o%@$WVA;Q3!yv~CB$iar$!1E?yRb@+kvUz47E_xX$nDqlKH z`wms0EGRz0e;3aaQJnrntkR-#T{w+W7AC=^x>oAe!c@36Ck%;7v+^NuMc*Qc)jjf& zl}ltM{Bd{RCU82K;F9XOAGlKf-+PfDu_i3}_~h4pCTZgSxZI_8J+s~*J9nB?xImVl z^IXx<*-+sXMGNnP(Wyie(tOtT`?w0W_--X`TG>|1DN)_|bLSPOKB7h3+U3|tjoN~h z7guswHyzXTQQRYY`S+}8LiK}GF|UbvyG54IChu|lApTDX)nrFGwwL4Lb?zJT&c0rC z9NwC7UVq;z?9vu4ki0z(`U3HrTl*DaI{)x8SxvFjiTl9$_07eCbR4z!8**L9;eOl1 zUGVzvlWU{_n_EL~9g1h7?r-NETi**hyZD49v!yd|6}5h}FIdp2@$Q?t4ELq9Wrn9- zq~ff%J9F5Wu<~ZHB9#63j)d(uzBk1ID-olH70=V>gG1k~%$9GJP@Y%nB7+{&>Z?`bSzXhsDou|HkFZ3NtWW6{&aNJ&ubJhwOt?+Lo5A~EZL6jc|w~L z_+pINNS%+ojt)}OdUO}aB0;teEb``MFzHQEm4Ct6h)gKVe3??cc?rwddp^LWWBm5X zljmi!5^vSJWu-=Py{^*wV ziui03cHtP@D#P;($WhMpX1m-_=8 zint`KyIW=DeL)F}ucZqL9Epwt6Pa@0jyt9|Al+^@n>4-9!uG;7Uu*t0bijERP+ep9%jIud?luZ;I!H7c%fZ~fL)9_Z zlA=pd5yPHb1T!FupQM45SwG124%;R%=1}9`9}1SvKpmBSjo!k9wb?zwGncP*N@r@y zi>-{mm_~4`v@ZXNi2hOOs;5;DM&?^0DhGFoM^X`oY%axj+Kn(aCUSfX8sBEWVVk`O zoxxOP^AAa zWTlj66<+V;5<^Kp?KG{UzQBLyrCI;bDi_N1X)L!Z%-QAV489wr7|PA`c_8t0;vnYX zKzqslXTP9e|NFbbZ>a*y+azpc?A<)+e2cRlHC$#(SMoFteY|rGeDaq^3sfO5H~SpY z=Tj(^w=wA{XbSYMxfybjS(C~#sf&3(vGQ2XT=Cen^O^Q6HQQxnI;|mf< zj{0KOp3I_~K0hAxHvIJo0%y+*!%uIp>=!E5I*0Y$^nen%9)h0v8yF~avC zz!`rmYF9}5Ug&4ORu?+Y84)Y8VMtK$9>WJ@=j>wMF=)xMHeTlV5!w9|e*e79u`Hh4&_ZAg)mWOvv3j#|eYSo+&-(bpv_xs(MN*nk= zvq!EL~iH2lqnH=WJa znk9N_f_f*uiL7d@#}-}!c8#V-P+3y@76> zNt7kNeW?T))zg#?7|rns?78yiP%>p8n;{1oPdz`ggF19thXuj3hb~kB`;DseeUI8q z=Z|o+B(PSRc}+|iW5i^yyR@MjSAUh0`XdiRF@J%%S+A47UaDo?m};6yf{Q`2;kYLV zuMgFsclFKNc%@4PU;Koh)ts~3xE2tP_FM6>eUuao8 z*p#wJu{ps=XKG1U-kykC+e?%sLZbmJ%i?0~?w+gehmzTX*rEl6+Y8(%1oiAX5}7sT z(LdUK$;e4zk}qzf>@fWn%SbmJ|NDWzTY~)GBK{e5f4NCv+3QQ-hUr1#tBX<2pveHm zJ|pw+7e~KcNaLrbYB+)KmaLLe%mFQYd)a#|(Ntg@U(N~A7o|SEK@&CWJaE5=+dMoY zIx-$AzUf|;&Yk2eoEYcdGi1fjik%>#`M<0{%_N6+b8D)xy;Rrcv`OFCB1@#*(qjar z)9(>IF1)YT#N9Z}A^GtMO+AQ~xOh2twl2z_!$d_z4G%jWyyqJ2@-w~G6zt_X$2|-T zauh^|Qf%4@=)i0qe~LR%_)AhtCZbjdZ!kYE<#wDm{0${WZYD)V`kA>RmgW1}7Mtli zCDq!lxiEtJ8IE_cef^L}iHC^Q$;-H>09KaAaV zYNK%phch2d+5f#vvBCy@!&|9sWKd)|trVx&R*`%YHF~@l!mEx5svlWtvc)w`(#icD z`A(@Swww7!F};2#!FGX!sQz)I-vayaD_C|QzU%M3!1$K#w=l9}PEO89I9j%cs?rAS zYf~RLQN?pOiRNq=>Bix5vv;Y5|9yvSghPSs?|Rj?zol@NwKDJd3xUZ^v; zJQxrZY_KfO{MhPb$@`h66c=rTWFhuGvRajbMd6XmRjNYe)@~yQi`KdW$Wa2&(#y%s zsas6mzAV-G-6rGy)>G~D9u^5PQDY8ikd)XiQ z?>F4dX{r077oxuF--MlX%gf7ik-nz>AWy3TL3h(Ba5CERk&u(wBlU>`Lx}l&l*`31 zS-D@p#jqX&r&sB)eaYU)CVADkC=V6K8S%KAqBETA^i!c`38k4TlWY^3#yjt1slf6 zsxB3H%?jE{zQH_>dDB}q2X=mlU%j0{Le~rsT2z6Q^nnC7k0}wVPulxcJY)l)ZE=7~ zm2N2 zF#|{9_=E%#z5bD1=eAW%(0laolF#6FO~2{#DtPO0OVX#!=e|WCa1AVKD$~s8wRbPRr=4kdTv(_@;ahy>$XA94m^?TeQuVz%yzdIkN3(-OkOB-h zlAw2#w3I+ooCIJXROHnO2D8n?k0fAIYVed{3-6cX1r}(t?OWg6eja(zKLpv|{S>l$0f( zrCQ19efOH?k7ttPrI^0^E?=%y>c8nzFZ^NScpI(W4~o5HQNRE*i|*piEvxl0L-@Mx z7U*yvt=Gktb3hHg3?@d+aw@tRVSUB2NipzuQJigF0t5mi?1|I&lcBk4iys3iN`XSE zK(e7f%`ix}jdzyE`g`ZE6 z0}v6vI+vqD#a0`pXtZm5l#0Nt$tuKKfycS112*z;fSx7~Dq$45ok3bK)c}Rv^4ex- z#Ai62=P!!J7!U^22;DZk3m74#cu-~7e0SZWTKHRTV1nTuWr_*M&)E0PeV3jBr{Xxx>a3j_>438l z@ln%=D(E2`zebAOZZbalA5asiNzRf^s!s8#>>(HfQhr0<2?fy}?VY`l3BCMuAf e@c-{kkA3LA)0VQcXmUOQe`F;UB`U;>{QoaUX@k80 literal 0 HcmV?d00001 diff --git a/5-appinfra/modules/htc-infra/agent/docs/pubsub_pattern.png b/5-appinfra/modules/htc-infra/agent/docs/pubsub_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..4453f2c9d2e50fd76d56410801cb4f52e9b6bca0 GIT binary patch literal 45806 zcmdSBWmH^E6fGDUYrF}rp-FHFgh22><1WE1xI=*8Bxpl$3GOr=T!T9Vg1fr}hY;M~ z<$LqqntA_c&6+>3pquKdI;YOvXYXBvC@V_gV3K1#d-e=RMp|6;*)t@>vu7X?FdFci zS6XxKz&E7Ns#36LWy6%a&z{jdlMxqHb2m6hM|W52nfbM{q`;8--1E4lh-RFyDL6RP zxIDOp<-(xC=(P_@rJJ}2zIdpw^l?j~h$Yl7y0vNL6n6zDwIY@6(tap;`*#5S&EWs*2hk5AG=y(= z^NdtP2qCEAPcI?=|LUdO51y$*Xb&9kvNq8tnerFeF|V?G=WK(1oBH<{pRCz8(9L3k z;m+aD5FgakP`ts$#>UQ*uj)d9&7nq(q!P#%4j3C7^DK(-3dr&mk^Z)mmzUQ!G$dhc zOlxdn;_|=W>g41EAtDO2o+@H0mX^I@q=AfcYV`E<%(nTwA0Ho&j*iaCp>I$JHt~!x zIzB#+6|uQ#>Nv2A{|0=C33m=w)LdL{@lwf=#4*E>!ABMMTV;$+PX3|icn6aX``-h? zM>B-HqGMv9ka0xLA~WWWy5Z%~5+-bScNYY0F-E)c_KU{tK!x#l$LdNTArnEsWQ^v% z#m~XJ5Ive)F+pGg9Lhit{fyD-ng8E=s~A2kkWR!Xgi3<~V#WjIArFp?m3I!WS|rB> zHoMa(F6~l#ILb6zOY` zWG@F3flC)mx^9Ft>XgQR7!jsS$8}LL5l(y>Oy)vA-=C4HHXmBFW*~w81c4Bv)S3OB z3_#pm5zi3Mi>W3wb*K2Fo4$WHgk?^vKrL{#wzo|-`(ln7qb?2?HbPm-9K6yQNE{C4 zYpen&J~iWy2%d&B)D_zB*e?cuYQ4@s@N5+HFwoACjBq$=*rpX0UfyWjgl!N4Q7JIX zvH_x^EJ#QbrwJa^(#;0pFUDI}(x2IziGGo{)E1M9qV74L`S@}A>mMtf2B*A~MT)nU z$``jlmTYvOzi_@xqP}0!1mE9ZuGAm?Tu^J9eV=e}VC(RFv%&1R`4rdl;$Tit7Ifs&xgMG0cw!cVy{GW^yX1HNdp`7c!2S6Q@@YGNiXd$Q#nZir(IoHaNC zlMSp+n1s>K@4Vv+6$@IeoiUi;gq3}o!a~e^^r1@p=3XGXvF2dCCqtj zr*BWz=4-=?L8vF5~pz4TK_b2w_{blCOqd|)&Q@aK$1&?hz5bF-{ zkb+F_#;Te>L#f}mMg$3%n3xQUK*9rL7HNJd+%^eI!xoFp9$f6LSL;p%QlC{H_RHHQ zt7~j#a4phZre%IFCk4T$aoHMza43o_BE!NyT*_w(>)#(p31g9|e(-IeXI&`lynghjKQCpxXz&lN`pD|9 zx2`poW1w%iRH`AU$8C?^i(bbqW;xgWEaj`!Bf?@Wp67AuRdx$JsKu{auV7{!E@+4- z;TtfJV)WFWJJ3a!8GXf-@P3-Nt_NDpXXA2LC%rEm`pwBYtjqGRpbu0XM_xbE_p|0f9>*l|p zUlA1OFh6Yx?be4I8^OoBQ%r-YVol9+d;^nT;Y`(|S}*ido4+Y9RGEc2F?in|)@2Et zrP!3IY(ys}7W^~Ahkd~%cAMY3lD_^@EYfGO>+hW-BHastW5Tb%O;vVUtzCPQY$SF_ zjqk!>lBzq%eN^xuycdzzr`yFCF)DT%#YA}T?>K;g-)*54>(;SPm+6NgA*02m9$a6d zQ@cNl61pI@Z@a&Ur8(P5{J7q*ozYcVw_G$WqDQ0&$*%3m3MtjCTVL^cRF7@@GwcaW zkkfLMhP#1)90g&{Hio&TL*gf!1fjfJ8hW=y0%kZ6kVtd~$Y{ zEDbINkGuq$>*2h*wkdfMkKKIkV%VLKXz%(U)pp2FeqmMN6=ujk?Z&JXMxcVAAW)#f zzt&7#2laGATljdoB~#nav(0N+cjh|%H&F;WkwxQmVRyfj!VoU|(`GNF>hRY%J~!|kzpTg|^Ny!Wd>Qe2Y5Y7$626=$f} zgiG~Y=gj)q_^ifXZQ18E{ROhEySThn#}!9Et>z2ZK@;*%T$kzvQ|2ck@HMOc=qEzp z6oXzuopV>+3(xVOX4;R6Cfjl`HMCgcvv7NsiRw>oC@2)Mr}!UG3T-TTAZJ3yzah&;_DIy)^+0mj2-~pePG-`U7 zf~Rr_r#wCu?pS$uYh_B~9KroUYA{heNa*>Ok@q!YojPtr$4GpRom-Y&S zYRGy`2XaZq2kWc!sZq1EYd)XWwT{AD(j$lSyB(EGTiVyJp*iTAu6Ktp^R}xy{-p25 zNM~o&v8>^(pB=H}fql2p0*YASzzw)sN%&e^o3W;>Min3C^Ki{@)O@PQafSHU=502k zjxC=hA+}>Bijomdu;lD)<184danG4>Jr^)&4-cQ$=Py>N?3y`CLjSirtujh-bHNA|F`BT*8E*M;q( z^Mt~dKa{jb5-Z$Zxc6qq(`K%UZqU5CYVxRlBeJUYRjI;7jEbd>Vdr?B+|=i`yMF_u z{LYrzGXNT(K2OterWEabm@sx0j0_%0jElzXV^1yswWnV9=SaKnpmI0i|1tDZ`4QE z_Ueo47KGeY1G|y?qMS)BnXVhvwXu=3<5%z-$$rPG&1LA2#IF_3?j$c;QleiWiLpY+ zL|_)l3_h%}>1SOE1!=i0evtA!mPjvuOPr}P{U+9uMgWx1uv#KU*9}Q|Z#{SO5{sS% zosPVJu~^-;5zy$o3j%#B6KZUEI+#HTLM1lW&QL|Pn)6*|?T+%I`{CBpevM_URPA8liIHH!lECg8k%$=XkNA$?U z8kzo!GfTkeaJs-0KPH10vA3ATD@uk@UHb+JiL^@j3_)#e5+KZL$@byWCcN)5yQa+5 z?eM}`W(>@3;Tk1ch<2E=lUzQ3z zzEQRfWG?l&zgIcXT|7vN1P?vdXQ6OSCuGD0b=ZQ2j|n5>8a?Z2P<$cemr9kP)-PwM z$yW897o)Ibr!cU2bUhfWUX!>)cKWg-VxUo}YF1PZ1bIBk0u-`3q%}O2$@O7hCQg{^ zTjGy6E^-vp`L`5#9GYX=O-(5>Lu?Epq4V=DB>^Mimj;rOl6i-!FU#is&goN?cwYTA z+WhmU$fm~feXnfo#aqpcjTM-L?u87^<^>V<92N=yZ+u4aDD<@BoQ6PsV*9B2QI9oN6&lAs6ZYCN24Gp3+YLIHxHI4i; zmX&l`!J($iM#aT5Wim3xDfO}K^^kS4P~`9gv|w=$XxI9hW<+T=(+v>LwE*U@B*lS# zY=p5`|8}Lg5IEp;Qdhcv%f-)K3d1%kVb5z}gZR_wwFr$045`n7lrn?~DWOi#Bq6Ty*{sHSoi6}5?%&mheL$Sh1 zw7Zr@!t0)+30}*43q{1|d9R)A2}A_Yw4taX`6<&(@%&_K6A3md32zKJ>&&_FAbs_Y z@88)Uk;HqcgQU93$$j$?TY`wVje-X`d{kN(@HbbuIctz{rzUv$tL4kI+t-vzE*Gt6 zD@U_cD?#xBLef8U5I6P;td>bfbAL2fJn7v+Kz{1L)U;Y6Y=<1cW@ctTsHO$8Vg%}5 zHaH{-Um80w@VRB8H`>0p?5#Rn)`jU)NFh{kv`H|}+-1BTfYyKWz1_Wj^%`V7NZ@Gi zPlNU%5%q7nBpVodE5qyf;_e`jlKHI1j`&-BYiG^Pm7)hu;b>nlAcLJH6uhAH-ox2a53P5345DZ5&jT; zQ}oDLby_~1cY;`KbV!AX=91@H1R4i5>($yldv8&DVH*s>({QLzc1+5%IJM_F?O((_ zB-&qN$vOPp`3+r~GFGVncV_AG7EkVvk$_Vq)%zB|9~}*2+(KiUF+E#hxq=(vOl4G9 z6CVl+3W^7FYptjL73olm_p71F8y#A$dDz)4{FJ!8IGkcjn{~HjoZ<7xr+63f=A%+q z5ROo(s{Xdz$7X90_Jrz<&MPViu(5(KX}YWDi9gj95#ZUnerQlAP$6!&=3bL)TuytH zVxzg@pL)}s_4O-3B;hlPYTH@2+`C`upi;$93^tq87|Ktb%aj7%x;p+ViShmgY`h8e zmWGximalqJ+jD$tI{8`61*{oT$7(rC*{QM{>f@6Yl9zowTx%TM{@~3wdsNF3_kPGq z4xOJ^KkQZ-4JYT8%?nUC$ zqv=)yM}|zhhjMXc+Ar3tjLt7>ZBE-T*2e-oMIq(Z^+>-&(Gj4myZmuk|XC(s;O zVXVrp6k7N7Bv-K^zZoC}{(gN46uMuEui=(UgGjG4rON*1xr{h2eLD4}f9 zsg@}=ufYMC84c;#NZ7a9gbk|Xx7uZ6A4{1*)^i`1+*XXP{;o;O%c~{JV4xq3jQ8Ij zx0O5nS@hg51>Rvr96|9;ql`=E_2a{Bxz2IJQR9Ap#7%!Z3#a#;yWsuVgc;wrozR@y z!=mCcRj=hVBKtv}#@GA?z06;WwP0%kcTc=c0SQb#zQQ`u0`l*Y2|cySR;y*9y7kP` zNHpDaL{BB7e;*UJh&@O+J@`|WgfuGPpYu9H#Zl|+(VM!tgSqMxU`iW^Z`8~rJAdaa zGMU0c_7!!EyquApaDqGVQK)?=kzY0YMHhTMQG4B>$cm8mKnKDmsrnjzu{m4m@cBoT zI5SJl*PGf9Sm=uqno@l?0BGwQnmU4L7@=blzQT8&W`8mo4dcmv9=w@EbUSV>x% zqxrSup7zwhb*YPCiOxO(ith{S~A-bU(zKeJpfltjRK~gp_A~uA- zKZwYI1L&L24(2`tc12P0sZw!nk;X`o(Bzz0p1-1_i;uMiXfRB@@X3uUiYXGRa>U}z zqgP{gPEPz3B@x{Z+0Na(r>e#Fl1BXMV4&9I8h2mj)|K?fbY7)Iy6NigNC{mA*G(cmZDv8eM`8M-%aS;GOnjHfNzpE#c>ncs!^zxH^JY>K4C zM04tx1PIg0GvL@%a)Dqvr;EHP@ix685oKQ%hGkl|=|3D1Ftc(({cho3 zo{@^xd3Xj)lh_-Srek-DlYK(6!dqp6y9I_kC#O( z0bXwJEixK;1w{Dz`kGm*9UoTXN~+MiuU+38TtLO~ysV-E0#G~t^IqNi%*6DpX`Ci# z#H?D%)3~DDzGz3>#=N{?_+L3;ZAAv(fCGGs_CGO@Xz+H&*Ecz&O}OdJGX>fTTtSyn zD!nJ-vTL5WFBj@X zbuV3R74J_%Ipsrvb*O#(xBx8fCh$4jTVFNw2a9Z4kk@CN?SB)-lU56=kt`w(6}$nV zmW19<%<-qe8fvWn0Ga5{A(O;x#mspKdOC%+gVZA_l@bd;+arp7MHroXf9a2)z>Wxu zHjfb~2)6=a6TAF<0=!1WU&5D#r2K3o zBp^4EFj?&gIu0A4c0*c zaTZ2)ekTxtGj{4Cx!==l-IM_e@E3a=O?U4q+JseciB?l@=pC)V^9}i(*>-=f1xY`Q z{oG(sOc5~WFH#!+QQ|nT`J6bN!-|co-)Ikg{zYPM0Mh<%7zhMy>m!er1x7T*RR{07 zzXE;@mIE-V5ua_*#8Zg{8bfFXfGWr;ZhR`AX^xAaxnpd5`+?1Nypeyum-R|HCA@^U#(wn+kMjWf~Qq#fi(q# zsF~X*j|5x~L?k7H`ZFkqh_=?!owOPM6Iw9k$>;5N~(E?JMvN%qGU^t+&1%uY|fG{`zlJjJF(%wYGnP?!Pg==x&0JR56 zg{Ju_7iZjMK1m<}e;mvRjD*jf_K)K~wGE#51VzYS@0R{I68_Uj*+CzuC7(n&zrU&g z$T!%aYa#k?q-rp67+JQ>#=X5lXNITm}+C9)sPc*w8SNPs2<<(YuvL) z0P7Rf%wl{`B5^IHtmZ)2gL-!f4vpN(uIEs?7e02N$NYzzsIFD4VOL{22ci;iMPbV$ zX47$BPSV?T16C?5Ex$9E`VJ2$r5a*tayIZu9fcUxYR;m9nM;Iu!_#MkE7Lso ziJBhI1qUSWJ2HM2!i-AyeJ(m#ZJ~5_v_#>uH_4gJ_e1O(aGvpcHg1N_b|(O0)`#ZZ zJIy8a0vYs4TPIBGce|m5NxXPA-v^#DV-yo2`|ZNE{X(6~VRLo3>W=`R$^cIiy^y4`MH0v$)O4~ffD?Z!pTr>=s8%p-?N3~w7akr& z1>&L|4}DUZ)sQSWjMbK?+!moRE=Z&pe$fGa$;rv2+}4B_2Xg_lSK>8Cd?@F-8PvW; z(ezKt-oczb>|1Pxq>?B}8YmL4uWvYZozLWaT4@zM)_LV7LW9z7UuV)E$Mi&UtLW&2 z-jsS~AChU(^7BBy_Znm<0MSTe6dAA_7-yQTcYl7sQ#^0;N+agcAf6cyk(Nf;3`j>> zZQdee)U`(KK6+7HZL3_)p*= zJ!9adP66`v$3D8Fycs-H@cNa zOrJ1gLG9muyF3+5g}CsR^%3`rR$g_OCT6|0c?%>W;zNOA{bN9` z%)kHT1$el{r(h{aC*|Z-e-0yHP*mH0&sw185Czw#IH8vTI21UGvZ|hM&j(>;2F|wt z7z8B0dVUf0P+`;}N;sg*NfCJueO^#(+40^Aiv6GJKLaL*vfsYAwr*mUWCXeUzuych z#Tsl%_VG4)jLLwdbp?TidkWltm)iXD?po`qnkZqGq&{-}WX}u*gzmK;1sSRdF@!6G zg>N7?06^z7A0+Komy?K+qn4~S{VlM*!15B{AhMqk%~qQ8Xgr?Q_J2CNI~~aZWE9$P zJWMqVFq>YzGAeWwwCAzAQJ6scryB1#bn$6+M1-@gp7hg$*Tld>!X025)%=B+mYEqNG8U9gkh`t0~LXbeL1^e}yrgvBGHW(`>Wpo4uLxD2vRm8Vx(S?qwq4 zVwNkdUr7NF9U3M#`gJuxx_qJkv)QKXe??GG&9G zxVS<{zmV5u_CiCKN|Tl5h?(iD56k{6%`W zzx?O|`Z*uW{8SFak=`>KRwo!9jl5p7yr4G2skK{}jw+Qk-Sn!}ieaTW>_kdXlPWW`nOF(cX?-(jwyjepYU4LSl3WtA9ycIUw zKqF=kK!2_ZTDZ0iJ}Ls}1eNm4q>C3+(}N*>#%ye&shT0161*y_3(2?2 z)}aAC&xr`XV*ab*++Sc&VS`eCYP^GXZjExVl{580 zRn2#E_6zpW%U(AWd3P&mLI*_qJ0Cl-5I&4CPDjxR%YiB;t;ZGjG?!`ZdO&jsn2u8Z z#A{4*1~dm7;I)C_}S{rKL=vLfcZt(gUXh3H#kBL zoFW(bE&buNtGaJs?%Q^T7cQG_|KjR_Sp*nNCFl+Tj%g-8UfE^*2+KzY8Z4+I3NZU@^SGdH16ynAmC^W`8P z?b!pOnb5^X5?hd3EwxTE*4K?)*7;)bj#k<}QsH`ZppsTxUnS#sBvgEZeE7Tq z@Djl?(jD3}&E#=Hq9CZ)a!-{kFUH5uL)^0spBZ>VP96FH%E0*k=DhxIC)Siv4S=R% zKu}O(GC!anE16Bz0&=(4#Q_rD@>cHoVciNX9i7DP#wYMeTnxa{Zvax{TUTXdJzYjo zMvF(D!P_mO_?Yv`UTUn0=ZF=(S+SkePput*QXYBnwM-(%^=m+Oy-1(d9N!o~Z>yWN z9nPP3JIZBx-!Mr=QvQe>Y6BPxS~M8Y&Y}rFT$yZ@kpU`9a#BsWeZv;99w1}5@0ZpA z4`kSP+M^w_|ulG{#Q>YH&d?*d)biPe2q+lybqwk*V;?qNmR%didwE*5HWMti}$DBNy53| zR~mbhK?e3wFIF)$kLjZlrD-;1=)%Mxo!vT20g*+70ybdDTudp#K z7wb9Q`33O$?ssycCdG$gbsVLn`M6_|)-ux4BKG!WpyXn1=pk8>J)FR&r^@}r*x}!H zb;WWh#CXrQy!A%*oQ;R)=YeY@TTXU1hvT~Vc@uyze>_m}If)B(9W6CYaXH{&b)0k) zC>3%u2~*V4Spu14we|2S%Y*#>>STi`ju&kbKCk?e_~H@Z;yxs>_{q@NFE`K4l5~a5 z!*F1C##|c%-;6l5r^xA7_?UW$fSfr&?#8n^X33CHjrbR zxzyl-@f&+dfB66gmN=I<`wGBo#nLBqy%}o;$>#3FP-nU08R=`1dhu1Q<&zF#OM;j4 zP3$y8UkjU=W|0z0KOk)upjuZ#KlK@ak4dV!dkc1I^K!G zI%Jj^7R+Jh0A5OjH)vSmRJOfacF!4C3VGoCXJw zf-7=!Urq646pGFe&N%N>)ypAKavFJ%IV^Zx!59{bDsJmlB*E|IJapYt?{5W_ouzU-_onf%91+%cT9Hn_>hW zE?rK2eob$E*fRWp-q86RmKM#6KWF%>;a%JI+|KPukun^z_W5ril?_)Z@0WwCkmd7n zz&*F{WU_$~NiE`abA#Et1q6tESDJT^j9#S{K-Rm5B;3mItq7P%78*h(so?dbkH9{% zXcDr4EVh77wm)?3+E8!x5)JmnKu0y)--Je_2PPM0qk@7is*Det*CEumiA>dDL8IrL z9vH#P*_+h_rHr}#`0-=!J5Xi1DN6+#X|3=ZnL*GI`lNdSO_QDVI$Sf$>-To znJ=>D6`BwW9t?M99pt0R0i%_)0>pN;v<#V65&Z>m9I3uHyXMYz-6lr?c}e@OyUZ`Y zT#30AoBSRa_<+?|Be;Y4j}QiU@EV*t?t#oPn{pJ*An+3aJ3goXf`&^O47f<+_wi=Z z>NPrHa=|w2f$T}G)FAnV#O|9Kusaba{j9IY#bbr{Cy+VA1Z6*EZ_WyjOdmtB4uNq&4Tq&lPnlJ;E&GW$N~k{Zc+iH1L!`3e`yXW1ZXd{i9xe*{{FujqGG9y$%Ne$dt_jPK7u%__0J*y6Vymc>pTkzQV zPl1I(e8Jh$v2k7Z;Z5PBs>FAb8C9F8U3JD9fPQ??PJpE0Yr-a{^%l$ytVL z=$S@UC7Eme&WGw~P*5PgJ2vvR3o8m9GHK0M*c}x~D#klB(NXfoo4sNpxcfnLQ}ssu zTrAS_a7^bf24!4U&zt~{OJs#f)8eI=$oTiQ7%6xN8ng_lvJLy!>S=|8&~^{WN41J^ zOz!m{b)65+=lVe5-&1-b7k&-rIhl*}UGi@AAi(Z9a25XBUvP^)wJO z)hA1UtfM&nRpwzQy>OHcm#m->ms-f^a{Fl^ygn~4D80$8W(bIys87s<#lC3sl&d{} z`1>E%vs@p)R(iFZjY6^M={&7B=O$(y<5gvQbO$uuc+c*yYJ5HMJa}IHdQA%HXR9#T zFrau-ZRZd~Ty_s(Js%&`sw!2xcRQUoFBx!d)ewhXNBnf5A|N-@g7|!4i6m5^VI=~a zfvNc7LipYb7fcHUc+{vAV%yFFPo#BAj6hjs5c->F$hyu*zrKbUQapd9E%H#b$nWdD#? zCD6NYVSq~t>+18K1VE^D5vT|m2v)Bp-eAENx2wIQh;E~1<0(pw5su@z`GEgIK8t9x zq&^_p6>niS&aBHX=OXrk_$0*=YqTOr!jE0U(DX{+6%l!{NTHbJ+{c_si_;Ueofu7V(saGg&T$wmy4ukPAW!e_T17;t&8jq$ z&>E);&Dq+E7ep6FN!7oLzlBGK0c%{8P)gJ*AIOw+`V|7#?V!ZVfp%LYx0DgiDR!0- z7%S}6AZH+J&e4AWnSLt6v2k69*0j|0@+D0h&8xiadsl`J7!kV~mo#|8_Kubp{TrE{ z*FXl{6z9wI{FJ|q0!Xa;Cy~#Ob`&&5wXNr_kaZZfoBJij&GVu>-j!nf9V8qS_zuWM zV;~!Zw?qC}9sEMF`(zWv-muhC=43g+h1F88E1dM5eXWT^?~-?5Z~Q44C#g9PqR2N6qEj^#sxQdb@%bSx*3U*Rom>)d!fexEqQTwY{PIV|QX6K>mtDo@6)ca~{)Kuv zDwUzxdpdKL<~ul1;F0|f&q6_l&uHj$hl;*#W<3;Px1=+1NZh#tSjI2h3yNj^89x zof2BDHNe^Nq|H*E|2c9TTqnnX(7Jmlvnr7)$;qAGMq|M5*ei{206{4FjaEYRPqfLm zUE#UzSru?U6du%9niRUVy`|J{ch0DXmelz^eS)^VyKIV)p5V9e1a|#Up!22%SOkfN zxN(%Rg->mz*&#_|Z%KPzKxmzHI4(0I8u2F<)JrfQOX;50XjLScig}?zMH_pi z@j;1k+LACPo`D0cRkG`+ENHhNzurDT=)s*7_=p=T>*>4RV(38ANPmv7BGN`NnEv3% zN;9^Z_l5jzdwdWB+CnFNP6-%5*7Q$}s9_Gi?I)-MTB~q?ZYZ{tZya{ttKkAye*Hsa zd@JaZGN$NEp>fbOtt=t818Du4*FI&9d(6&<^R*@K2hqF$;ns>S`)Un+3(brMYrkNi z0>Dtr`fwBanf|v~>H~`kUYAGeDOCy=nLX*QHIVI*%<{!5L8P?%qnYx|qJk<~$Qwgr zEM+1P6d8_74>w5FgX@2@-K)vLrC>UyCl7KVk;ksqLahy7nJ>~r6 z(BcSbN|OgmY2mz_RbPL0 zu@)WR`E|M`aZYJF!Xc-0V+uVo>9NHoMjuJRCw(uY;_g#sqR6-;*(^`_-h%P#zeEBo z@{cred4`VdnMF@|J7KeaLR5o%MAlTT%?!KxJ4D`8*EUUa(l5Vc3t)S%ldcZrgL=>T zV9EBXY&d=bI9JWiTva&qLnh?Gz+Elo!up9wjC0CA5^cIJNXv8M8eOqZv&Pc)7PV$RSz{6`XyBG^&Z!9cPY`m*K zdfGRpV275N27dsOroP8wqkO3MhkP*k4713r{UTKZ_~9c%&eFCH#(+C5a2Ep1!G4`e zcVwszO#Y@9W)moIVpYjO!P+661bK#ng=I8!@T2)#vI;ljN#X^yf;U3$7AVv)wl z#{2SL5_wt=0Zv-OhG0X~;i()Fn@Z^lz(o61ktTO8s^fzj?swpWs6gHX(Z#d)LzHvLPgOst$o*N{m8|CgfsTXPvI?07@Pw7#<1PD z%F~Li;%j5IPOY`eI7R;XH=nE?(c-|PAE7F$%OZuw&XgmXxl`g0YhNQD%9lItcSNQ5 zI`U=H31FnXfwK)AgoI2Y6>!NS1Z|jR^Y>ThA?q7uIVA!LOgMJZEJ!U-r0*F5xNJ6W zl0NnZbLKKs(g5l(LV`N$(dY7T3_H05aQJX96jFKyG9A@Zg)3~`azi#yX-J*k7gA1)2#P}Aapc>{DUwto5 z4zI8K_IESoccaU`T0Q$qq}ptBSWU%_|0_l?HsHc!shPCI@Wg^P(pkw=x@Y@wfjyjF zKxN$M^b6$WhHGiOMIGg3jt&}7U}*BTNJTIjQ{PbpHuMj&=T)}Wfq-%E)Qe5cn;HZN zNmjb{>&=Zq-8t=k)`KJ&Dp?HCAU6b@`|v8)h_mLSo{gT71PS`Po8LwH-f* zzJA)(_2B&JgpLz7z2V2g*FFX4unfypY*@x(ftBnf6!|@A{z&6a#ztdY@&COF5rfHS zvc-tzUk!2C>R?3zUF6P%@hd8$9Elu9XnOQ|omr5_=O;a6<-OH9?I3VbNaM->O*NCR zkm^*%AZdPovtQ0W0~fwO`%wF*o102%m5`9I-kFMTfk8a+rG9|kWBqP^Dk0@8JKHmV zah4h|#%!n`hO+?w5!-qSso#K*lwKRa3n?jrSY3=o&|M~wp-OK@{e>yTYL=Fy>gbK; z>pp!vrxRtVIkM!`AkKdcd2_+FJ2DUKeIbQBxP||RGL_n15j=+u0MlbRQUNwR&ee`= zzWCNPdSD#IQN5M5&EJJAI*M>2k|iqzj%@3J1h)R7QJ#S38YRb71MDBl>V{I^70YCV z{hjwj*O%8ju3z=Xij#|68+P6wquaDr2tIdM@;92pe!H52+EzMoWB7uK?X_KanrN_( z3Ha@T$*nV|wFc>9<~i4@aVH#u83q*7dUgq$=tKOpJ4a`>=P>M;Kiqm7zX5gK4!^Nx z{Y<2f0Ed?vjyp4a#j2O;{EE1*DdtXc>@N?axqrPnB+%kxqF9@r)ETDweaLYPMVMc1dn zv@~sA=;!p_lLfA}bFS}^D}4_i-nM_Dm&tR7XY6eFD7E%kq$B0o3;k7X&{u1g6^PK{nIT?x#0^#gTKUEvffgPWOo5S#L(wjK^DLMbIHM zEA8CotxRBc>7~``lfZ48?Fg)c7N82B;S$34@={K=7L7}WYWxf6Mx7cC2T3^sFK6d^ z6h|UXQD6#h2Hj0XSbrW=ujTCaFf#sp$%$Kk7DuZ0S@LayeL49L%AL-2QzcDO1~0{( z7hlg}`1ivTLgN0a0PYQ86FT4_R|2$vVLjn<`?6{YlV{=t!`37bTnk$LRmFcck$uWo zI~H;BR{ipIeqh6DoY7puI(^H=q^}+)Jl$_Pfr|y4CJG_^VUbdIX%mP$52u(ErC=Mv zdGA*`nAq;C{3yt%?#90V$HyT`^qQ)CJ>N!k#xLdIZ_zd}R~LEnwvYcpoGlfm81 z^^P@E-2ZlJz3S#CF`R)47(7Hsfwbej%Kkhy{c(56g$0lNdIqN66(gZxqhVk7T7TeN z<^Il$4Y=z~vy!~aWPw`Q?gv~W{q2&9i!)>ycf@}PDraFv)i|nIpP|c>mJ^v1M%V}s zN+n7X>&>u`Q;}=^ePo#6RnMv?#Pm$Pn41(Wh@|@tHErLgQHT2iMi>Yw3SJWpt%VcI^>d5)VNQw0Bp^Lm=5aP)irY7}g z^Zn4k|Ng%Kz|~!BQWhT(>3!lE|0HXuz{vHrDW?kk+hQx&1~%4u_E)c@MUVi`akbGU zG<8Dx6X=^Lw%uNaH1e9RoKqqrgcR_z_>Ij4YdlrL9VM{U(n6|9*ScM=Samwlk=*Y5 ztbfpMM$_;5I?E|`mY-C-8A^THGQr$GV(V)u#nJlcfaQ5>0@nlFLU3$===jDxg(dkT zw(Wbvu7rD@IGyy8nIzvcaV~)2L)eGf_-KbrMN*+3VhUqlyJ{5mrUH&g(UkgSYN!zy4o!;Cy?tpVUmp!HkY)JC9J9B^K|uy)e;{Q%g` zEM`$*XyNLNqS9WJ)Rv9zHb0!ktXL!$ZBta1!tEp2g3le5Vvw8~022ckpKwq@EQzBH zdrgFcQ{h>)+rP(MYYJ}bIFMSoK?@-~i8u1EF?%{f>|0SQhoBu{1EOA0 z(YN1cNwL8gb4hD)GhCT?10~wkYN6}!k)xoyR3`rAn9FflW$#yC4K$Y7KKQ&`Rd0)JuB)_Rlzc0`-s{;mPQ;JZNH25TX~ z!fiimkYZepis5;nK$n%2zXJbV0QmPG>P8ETc;a&TKo8$Y)6LlQ)E9F653=4mEXptH z8kUk07+RX4L68=Z9$FftJEW8bK^h!796$k)?v|7;K?z9-5s*f@yWcbZp69#X>;3p+ z)Jx_*=RUdjUTd%I8Wu8kX)z_lxcqRAi@5=iB|}Qvh01Z^Et<-lP_%IzKTJeug`g?Al0h{P`=T=KLg*BJ(eM@)wRRG zd&rZuO%jTfa{$<2HvpUyVFJ34+K6L8l=}D~jMVQa~8ZQV4zr zPX3f`!-&|=x{q(dGM>h^c=h}j{@PEI$=Emj5G~akG`jZqI_In33Uxp1*(;RIyl-D^ zxCFCoieZ32)uP$)D)Q@TnIFl=1Vn-^F0U*0+$?S7OAHum9&kX8r$4+Wfl6|dE4bqq z2P!K^zmboTu2U8StuFv(F81hXRgtbZMr7YcQBukS<>z)RLqu3W`J!!j&QvOyTKc*1 z^N55cSQWg5RHK3%^kHdzP++l*?O#fiTi$>CJ=|Ss#h5zJtg0p~4 zL4<{|vzi$8kB0^U@3NweBp2sMjR4v&Haz6?P^s%*`&e*v*daTTf~HAt2NokbOxpJ% z&*ESE=rC011qrfKY!Cr3i>TRIt$sf%|Fw?_1=o-S28IGb_P&1HC42v{oMz`})`=YG zDohFGFKU#kqP%gPbvH{M{bOEIUf0O%ak;Kb2GB8zQ@+NMqJ{;w(>?uA0DMCC_DwO; z6LPEqob@On7!0bx=bIY~>V`Xii!|Hues0y_Un!7)PG27q(G5+9_>1Z;EmR2hidW{A z;T1fz>0jJbDpp)kKY7l?Qdm6C{JeBGxlJ)YN2-C{yymydwq4w%Nb0B97RK;+FD5F5 zchvF$rcS??Q@BPIOUI}MDQjh7eaaDOqr$VaAZ!+lu;4EL<0SJp?5@)FKz3x}I}s^O zv`?&Ij^@#+-Bc}}!?=UFzOPn1n+7RqWl|Eo`x?vkY_#ltm}+ z_G}ZG=#$Ui|JFroK2LeqUE5?gw#+yFYMgiF1u+-{zW}9+okuTc_+A+vQ^ncrX+EyY zxWUYITr;&e;u=^BPo@G=IM0LbT{{1D)^FQ)o^sR$)JOPf>t~q5&R^)%%(4k`W&8Pv zP}l(yBs#T7gLdPr+KxZ8=|qf9bCg$Nf>j8RF@s!jsQW}B0t*5_I3T&`8CC5@EOSuZaBc1uh0uU-Eke)(u>d;9Ng~dSLqhy*!`PH5Z)3L;|LVWXRX%RnY!ESw z+nXzL&THH1>6ha>*P`*dg>w5(wX|7&?EYvFP=6|M9$AFg-1 zAEJ?~_#Eeze%(Foo_D(O9*)BgL*#dcaU$!Ckl)-X%%$h1!ewcVwGz3~D3s=Z&~h2j z%_{qM*RkHR7XzdiKdQ7{W4S&Z6y*r1FpZ{N_dDg14g}f(Qbj+L_w{!J1_09wP{T(` zWW0=BF7~t9YkwkH?)xUevt5|`TglE)R^LpV#lKHAd7{DIT{*mY#33B=MzTF5q(x@k z^)LPLx|b5qypFvQIJF*VWmVGCwcqWdymmAyoORprB@isK*(*hkPg06-;8jHHIvnw8D=l!F7D+3G~Nhji2yYNS(>XqLH8YfF!Dcx)jOiZoiS(_b;r^c z+GX%*H17o~c*Qn^e#e7;R#{3?3HR*86k%{*@HoaiP{UF$QYUy|<(Rw|-2VqgCsk7h{TwC4?Q5tKH}dRjRLdNTl9| znS-#f_xWP@C%s3E#@DmyC9Z%&w~% zbRo=#fx+vz{BU%z#(gJKw~~q@tI6w%qAkY>t%CQHu(9{|lsRY0x?0dyw4AV$#&IFp zvL#`tHFwweKmMxAbiKYA$sQI&@wG!vkxK9f{`QWF6Zg@AF$!Vl{c;pv?0hI_0jmx5 z^(Q>5`8Us3xi^K5<(?~J57>BsuPCy2Xjk4{EE;roGcOce$@klgFAQVnl6?EDhom18n8|dtFsEXHKxYXlJQ#?aPak)klvxI+9L)JikqRd-ZzR zZdaN#oe|khG3_AMB&K|uqJ``*)tQpm75NWz^6l`}=K8F)d`5)f%-!QRpbJ=6Up{v% zvD0PQq|0+W?6l^iIFZPFK3wvxrlX4l_wcCyHYp(?BqruTdmTGzG-`Z72a=_k?6t)I z{(VB++?r`9*^b)iP0`iac1L}tM0g%0-sn|Sq35r)3RL9d-2(#*XjFW5Ot`b;2QPQ$ zC^Bz{YDx;Oui3vi8y4?M?$nNr6l=2zI8Kv*oC}EwcCh_*41$5xZvj-5^qV}`k>Un< zc4Nhbhc@ejTY?wi1UgBRL$>o_K3r-?r30wh9QfI3H#0nY$$BHTI<|G}6K0|FW#@RM zy5)vQyNlEhWH03e@!|k$`#zA^5wEVP8EueLc+hl;VnWK#KG)!-K>q}aVQYHCr(zRb zammqaK>z$3wAN#qb*lMAg$skeSudqmGD6{M2@-lpqrpdw*xdzvCuAQ-=Pz95gRI=}+U; z?{%;2K>SB(SHINt#m)$31MhC^Msq^VOwEfyULJ;;uK8$=96+JdE?*t-R}{w3oM1&o zMZ2EjId2qyUu^}86DOPNN1?^db*9T{$ET4tBf9n$WA}oa2#wmXj+n_J1-E^Y$o zhbUBwqI*J>aR7RHA!}n}EiO0H;Ip`Ycd;r!8mvJYig<1LM5xFuw?r~Zs+X?hS$a;8 zFGdV}WMIDiYg8z|%cSRLMhV^t|7BBCFnKu;Lmqn){9V)N?5+&yav#+eKS!}i80TmD|9Zjr3syhz1Q^#-R1g)Xnc>=kTXeB`U z!_4}_GdYxcoQK%itGKt{%Ou-QXZo=Jy`90l_**Lx-y5csiADKv`DVEX&JX9f zPgsB>s@Iv$HW)FV_aRCS#5T~|&~!Gpg{$BU`*=wTL>RYbD2%`0-9$LEFG0vsf2+h2 z`}fXkknD{K(pD8!RpnwS+tyroy|MC|IFMm5F-%i>>-`3~r|`eSneTuw4hg@_fRw4J z=|>;Hxc`QL%DBw?pFu@Lew*u{Y5xJxuIpkbIpCIK45;UR0IW$Gq^2HS;B_!ial2o@ zPikn;;?1ijvid%&mgaFR4*ctjy#6z7`sWt2ajUC2hBX^*F9T+TW>I;0d3=sY{W_p}CZ#%uJ_<%EJ|koI!|TqrDN=Y!VL?$5 zG6h__qDLxecGg(H-tBrSWqW&D>IuzjM{M|4r{3q`7(h1f_&b_#QgUv<%P+Fo23wCg zUby7`-4XAfd>9Ey<*`6@YP+zJRZv(4z`h&^vZS}|)J6i5P$wm(1UQyd|KxAoA?lv-gh< zK`8C{$ZOq{s8nK>&xM{{S0Mfb2Q>YX6SneK^KCRnUtVcq6B#tHeETZ#baORU0`Te; z)YR~wdT!=rBu6eZgb4xEv$Dqck;5Yh4s)&d9CiK~+!y9%ZBz*~f`w{sk z-G_h9z;}zS;TdJvCVKp7n*eqlIJ>Y-8pf|I5j${03||f!+@BWSjXg- zvAfvfQtms6lH(Hnl&^;Fo<60+AY@Vz^2~c4XhxV6ynIJ1-cP@FaZG5ahXaSb6cgKt zl`0C+%ANPS_eMcYO7W`Z$w&L7_|q%kWU+5WY{0FSml$_R93Mxx|6DPQu!;1(bQ3XE zpm+JoA|^bV`Fh&EKLLo=aq}Sc-SAXC^2zzj{KBcR<`bdkqH033FV_e(?dkhC*#}~B z!A4FZRk`VqoNqG9!HGV4OIS z(!=A~L}wsrlCal$Mtiz9|Cy6a0*sHpLe`Owp%4f|75{vzySSDaSW(7qsQw_xZX_%C z((79ksRLfz>RW=i%s1Y@UnCdTiI^P&b&*T#*G=kPJJb0LRE>X0hveuO3L6 z&6u1Y~Cr5HFmmLdp)@Fbc27Ss8$6)bF*0KGw``keeD4&E2No z`KyWt2e8?H<o4(c}4 z3Mm^XkBC)=)s0VwNm`6nQB(i;^LY+0(Aj>0yb2H|diQO4N?=`3Sv}l#5j~I0hZ7co zzn1%#il=s!oSr6P$A0n(`fb@cdGZ6~@6=M-iH}&O2-qj6AI`s?aYP32NCW`fz0zfM zQ85z52GWkR7}xt?To#JP+xs!<;WcS(kMOuSL zdgB>Mr#Q%*Twd&>cB%#eLc-L4+G_y2*1oJH7?!vyyJXO#fcs4-fG0iJSC(cXb?r$P zO9e+s@*ye+)%uO0;X!SaH2vzBYImp597WTEW1g;hjVKtJqsmlcqXvA+DeX_raXcL_ zqqg;Q84S-}sBKvvnjpKSDNlU$5+iTlJX39!VhCh}f&RloGrD;G;t1O7~_orIaQc0f!AG5gb;m*wxhVdFSkI3Dw%w^LU~_6sP?DpV>UL8Tid2AEbhlaLN6y(FrY4u5nEzI{UBBA9f+AP1Nl&G zT}V09kgy`C^QpJr$nrbMib*#>;_GpP02a%H5~0s@m{E^X8v`60WcVB1j1c){vUBZI|h_sNtY10-mAW zOn!Z}UxQw+Nu&VQ!J7>=!+NLmkf=w$QVMWcHRTOPF)Cm07?Sd2{Zaq%VkApijd}c( z!9H;r2x}A=zp6#7ySeLP(T7Axnaff}+hB{lg_LtNk(-n+pD)E70n+$4nRUpfp6MHA zZyYo(x*pgM_^3yQKCEq%5#n}+EpdCMC?aPx>zWZ;ucvoG19O(hGG=Ry2HwAC z7rImFf4fD1MaVMS(XX){g+h5vBUC^%PFY(QTj%j-w5b$}+w6o8>&+8K9Jtt6UqmT7 zOulGcLxBkv@vetw6A4;jXXXR9dGnL#nea%0MK=Rf<{oc~wCD@z#IRk~rb8?*TEecu zlCs2?BF0&3k+TnMZCXKk6HJQNHWexuON|0v{mjAMthNMz?+Ml@o>E%t0T%4T^>!?fOtz&_I;T31`)ZpuJO1tXiQt zE|E*-Ek#8+soM2}Dx2QgP4HF3Y2?_Zz1{)y7g43h!z+hd=L$3ilZR;8QOO5SWxWw)y zi9=xWdPAauEGqH!e$u5VkF;|#;{;8i9(kh~$v!eCG#Jr7&#Tie#YWGk%$OrJ^S<`It|rNx%IsGLL%r69Q$8ed8Qqs92EBq{1j$8{S;DCS{k1R??Rga9zaQpS^ z+T9Tuy>wKl$>AH%8o=2{e#_}wd&<**+Wz;&s|~J!$;MECJ@}BCJFym!G+r0y4vdh(>=jTJHrE0q40{0xrc50Ipal3i?9Z9n-sJ_C%## z7!~@5$j3NXs@JMmtlj0RKqqDYWQL=Y>ZqIN7s$o{;oEnp17qEO&j-9b|Cn$?TW!Su z;DO#Lue$<5kyUSr=ByN*V$6N z5DM`US^AZDl{qtj8U@iw_)>uIM#=L5m9>Etdo2*4?s$m7$Ip*cLEyE)%bsXyUpaeKH?oaaNlW+bW6)}RjmHxE0e$Di@co7nPtY2Z9@fKs?Nq7GTZi~ zvNq__%nvOm$LfKvHvllSn#w`wS(o94yv$et?dbLvLk{Djq#7ug=YXWIu)LzldC_8N z5-b7?V1!lzVR32+KYdSOUUpLdp=KPenM|8=G;qNukG}{-&Aw-^dN#Ysd}lt9@LSl= z#W~3EgH4ML5Y_0i=6mD&$(T+G!(rlO2>zeT9RK>ZfR{S-Af%vPqEAmET>it5gJYm4 zKJD#7yB172Sn$VRhR1D`sdc+VhBS$D^b*-m6SVmscs4gC=+KiikgFlTlb_F{v2f%? zD+jYA0zuzmp7nB%Q8Td2uk*M2PQczMtg9ti!m-|+ipc!lR^sz!*`F~!L6^VVm-EjN z5Ws;3vOj{r9DMFqv>tuyVCUotjYS<}jqBQEuF(l>K5IrV%oJgJb@`Y^P#5f!Dj!|l zPx`RnF+)Qf3yiY-{PTILvk&EC%=tD8f3M647HO4#T^EBjm${kr+33IiL3GW>roWxv zgcEeG3zNzBnz$Y<7yVLngN5!_UX+9S1MNz_THjE!JM6i*LMu^;+x^LnggCh|p<(wj z#rMBMcI!L_|KAtN!66}|?qX%!3;tvCXogWXn(ZkIzL&%;bbw4b;?%fl6)B^0tEEfN z6pS70TmQ8ne-*dDYLafA?UCY4QOV7joXoCl^y|W~!5T${Sb!8-$HZlfGbABcIr;<% z%RA2Qdaq?}Gsz&6ZPL&+e2V#dzF=8_oZqq z%#|aYBa?<~oyFl+3Ixqx;3_QRB{S$U&Vei|nF>$s(!gvj*LvsyF5)gpLc2m4YIFVP z9uT!D8^`%cds}d2s^DfQ-|YY;&@~0IkcBxc@KKwT1XBra51@(`R9<_(gXcCOEqAn~!dU!SPF}cE!;fEC* zjcr@C{F(GZzBb?+H9S-y%k{_Bk^f}v1l|(!n#@bPzZ!7(fUJ=73=B4D!6klyf=c{d zEUqX;|AF6idjO9iMUDX$7WT)`vXtd+YTG|^hd(ml@N2i<59n$Uh(*Lx$Ny>S`W|Rl zQlAMojE%_}rT5MQ%4AJ5`7@D@o}Ot1GupZn5(?+MXQ0^o6p_z*CoQGe8#viN5RZfX zG16)7iyUz^{>zO%0tkfY@U2|6c?1kb7Q~r~@T6B(zkM5oK-`sm_{c~~#7As03U?W{ z`fL0m+j z)NS5(2F7YCBZ4Jv9~_n{!c8Z2ba5^uQNe(fXCn}v`pW7OVI5HzV;%ag7?J%IlqB#W zq50_VT+QcG53Gf4px~Kg)fY3QME~Bg2O{kCSGetFs_r#0HFbzW1|4gsvj3}_0qQse z;^X>XMYSV} zf2eR?B3egZbOgQh7AH_>kB1eu_c+*}^04mO=^f5?|UWA>Hc%CLIZF^g6 z8c~1|svv9GyMNGcrM9w(oRg+Hvj2LiKzRP5A5*x&R+-jOkuf`LP;~LqX!mF|{zv5i z4E!Re^*3tPzrtImN%P0GFSjTM(j}n7-hxD%R}0E)W{pYn19JBr9)MncUnnXh9fcof zF}*;g3?3MGz@r7Tk@ul>EYd$n@Lt&PLi5?>6X^jnX;9A(hyR;D3v{yUOXe@m#uf5K z_R_yRoAAm-&kh$ge~kP;ZJV8V951M;+-*9_Ns%)jL9T{{P-xA!y}j z6|>7_V@X6#cJ&dz%5X$bX1$8Wfa}~G8VojPin1|p3(;@1yFYo$s~lUXJREE{o>Ma% zqk7+x3pEGZAiVy6COb~*Yom6`^(ZcaM_i&!gY1F2VlOJ(g`#~O`@IIpUz4gjM|F?z+FUZu@nfwR-BG`xNIf|g;5`RhE zEeNTJ%AZlWZzMG&6ktOyAg5((HiKv=y*p!P%hhiF3AJd$4_rpX<8)|Nqb)8>0xqQi z;n4g1t_ja5+vetFr^YjzXawRn*4<1Svfqqcf)u66*HNmaYip;56~-d(IdpJ9zH7(j4dm9d5%(mlI35F$#OX|S(U-aaww0=v_~ z-)`^8wS*OeR>ET_ko5=LJ2{{TL4p#Ag&Gp%j$V|$JJ(7DWInn;OgKI*EvDhTdj0pF zYj<*mIgSSmR?x9osR@PRR+`qG+m;zT(LV9G0rFYExM0)Di@*?9CJ&J^ha!@a!ayMj zOAso>!o!mZ-@TC)=zymwh$n+gJHmnp`VX&}{qEbRA)P$!|Z+*SfB;|JEq30d{t0|r{ z4rP03DToU$N|XCLN^_I}3gBPj!9xRq4)bY(dc(+V%Vi3&pW6YoP?$hGci-=}vuevQ=+ zCHxxNo@qUUK+whH7cSPq895nn&LmI1^f+Q;zj=_odmj$>kjekN5RWE$F!swi7byq^ z;Kmc31T8R#3%sbAfwv++at$u6-WoealrQc8HVNbTYzZgR3Yl>Q?vW~F{ywO1L;#fY zyiBB@Ar}o4Yamq+7v#fCM_^z>NX@D5N9g~$KWV|HXqTEK34DEA8-E*3Ie-VcL0b*?t;gOj-u+-8NI|rF6BO3+Hqj?i}v~}!ED};VdI%Gs{xQ~B@CJ$#u;RO!q5E40z z7cDl0yoZ1}mVrz)xH5t=Jh&d~lLUj)9!Ev0Q0Th__tI(v!pNoZF?c^pMs_GxN^lTD zBjEYnQRW@Ju<%0=(H25;lbI$V;zR!)NCj5#Lk)X1$VenOp%I9o@+E^RuempvAd{Wk z>ESWT*XZtsiK)FrS_NZ2_SvJUEvE_hLNL9ec1i})9l~HA!r?J9su;CQOxD-~Kb7t= zA~G`Y@81r9pTSKktT9r6PgGme;bMQp!vd;}pOf$#6Tyl2xy1>!GK zK&)&OP!kLNc@uhS?Z@_kMv^7SK8y-tdsPU+v`7gs9+w3v(8+V?PrWcsCeY2kzu@{N zx1rTi7e65ZiFKg>YayBX9Yevt59o{8eiw&RUmQO*q8)Wjes26rNi7**Z1Ui!vOc-2 ztZbGpG%RchS!`%;uJwq-3NqW>Eo)K}2tw%xpd^Ek+3yScAla$MlM5pAr zCes(^c?shdZ_S%Drc;0jKmi9+FR)El5&ZNfh_f?UKOO-0(_(2nq^l3;9I~2q0UpA_ zOcG$jhM)*DGBQ@Z_(7IWW1kZb*4y$(zVe*FBn1UUf7vN0eX|6xCT*Rzi@#Jz;j#Jv zH@E7NdVso}ya@LH1E?sClmwCccei_Y&(mb$!e3l0jM2&cYuNwM|2Di81+UFeWcp>| zMf2FKEe1T$C_lKK6%U7W?}`%O6w2^KYCsi1YS1RZM7eN zM!&%&n{Pvefo~bYTh9yyKki;rlI6%ruOVat@`;bZz!x>+&-s=aH)_2RIQ%5K#|_G# z#8g)DgcC0co#~Y720MadUg=}p=0s^Y2rjLbx`PN5H4q#6VYHY)%2C2)0m9L8`58_zftAG-0)ZWFp&>K1ib=CPpKA$KsK(dLCUA2u=Z+xO%ZLO4lqG1=d>`$;4crGHC6#_5lki0#Y`9r|#_@zA(xgD5& zbrpM!Q)20}0i77r{qKP0eDA!^7FQXluQZ$mRL$Z5O3G`d&zPDZ|BkjC-`m&1v;c%h zb3qWbk;qac zeSbztpJO(O%XP*y34cn^xgH=8>OOi0kb-e(T0+8L)=WRGkA>&Xv>8nCKzfMdXO3+$ z3neyK&e-EoMJS^J-=a@I!80VN_$+a{(Y>(Ax#|O5z&VPaYQUJPQ1K6MJ&@OUZ-ffr ze_&p?e7`T660J_c!8DW^8Y?AyKfwz`e`i^s30#bdW1r$|gm8H2MYMk`HQjHDV0j-H z%4`{e-E1lVC6=nQ|EWmCK*4F&CZ_({z~PZVkLJ2&&kwg}J#jP>t758}0!qL87rVkI zP}F}{TD;|}(i9+CYQHEqJK|&{JTQ_Zuue$Xu{CT4#lf@+Sg)e}6xM&ED7uUa(-nFl7U{RPxNFmPT;Tya*2rFoA%K zrp-`VBpN1u(HUQhQ5AVPKDIP}pFNn&hpqkFp@zR+er_!gU~rH~)p%){iZ_S0GgT$i z$U)AOD$i@#jYXO=X}Ye`Avq)f)qC_^w6urTSH<4ul!ZSQt{+ywXOx{}e}%u1WaQKd zFoEswlk5#oz0-^B@udp>4rD;cehzJg8 zBb@Edy_3D9PhrjibIT3xtliCY3HJxxm~wzB&~GTfkdPS6M@~U#l@>ox$8|61)2zw4 zuSk>^MzYTE@3NpJM_v}D4)Y+9_)SY$+DUISR%+3?YMK{w1_JU+$6fWWIH9T&(5uyS zOe_L*0JCUqo-#laXjPr#;Ichkrta{pQem2h>ugn)-*8OK9b`L1nT*I^mGkX=5ti^$ zjPRs@E+Tbg23@Yx!dn}fh&k*~$MH8-r0X5$Vnw2+&5bfE){c*@3@-svH&jg3`go*8 zpwgf&$ zD!k~=NO{Wiy}=h>rpe#F)G`{`EeyE5SetDoY-N(gi#TJCap|Z!s?5J%sJ9+GS^Yv; zjs%c0w7hHt95uVLbWbj~^bWO2!{GYEr|F1RgCSzo2uE$I9Da=U=OZmj1Ka736t@Lt zK=u+o0I0QP_c(JxO$S;u)&9GI!Td=K9K&*LKiDKJzEzhXVJ18<4a=}3xlOYXZcD5p z^`aqOcNmb#AT0bn4B+I#$FmoQnVo_gBvRGOcW(&fY|)}ji7X9vXo&6xm3>Snn;9QK zp7BEk<-S|w{*Hlv=jA6M983}Gpm?iF+-l*gXyYgF;|%6Owe}~bApgxX7&UI4#P`tL zdiqd9LhMLptuY3KyOq*ypJs z_fm>DdE=_a%{$YK6+pG-Ps}JeW)VVb{e=*J778824zh4OZozYd^AU(h;4mv89==K8 zpJJ<%a)|yiilcAFja|jrilOvgAp8%2sX5l!zF`qriZUtR?vBjbW>SX071r%!rj!6NFcE%giacHh;PbCn!WPYGCKDM<;GpK|cDUZdtBUFeu$}O*VL%}cG=1T=U3u%;iF*@ul;>pga2L;xJJx8n8vKf50+aK z_=UDYSX6q@Qv;0o)Pr_@ z&fQ)zyExckvR%zYb06*O8;W&C96zme_5Se~A6c4ioC$w_Mva1OZ`dn7jtsahK}>Qa zs2;RgdJjX-l##p=k+@mXgh3$voK3s(^VPRz`Z+Y-G6Q_l4CUYL<2^8mXD{n& z68AkFP!%pLj4iEufQFbZkXn$@y>G*%czzl|p~0^5x5oq52FIO|mqYm1lxC8CtN#%S z&%8XZ`(y2Z^N|0ZOYrZx;oZP<^w6nQwL==eC0W^i@ID!ZJahDiyq!;;)@KNN8d^$y z8R+&046`SsHe5v<+Qs~fCKjQ6nDc%|s$$0@G8{Z5+`y3obAF|v*z2DnNYn&qpU=-z zL*VfkwF0QuHp4v`ei40klMI)N5fAw&UxUmz9m>#!1@myyk0&2Adq46kDfS-yVMbo8 z7mt6Ah9D65i0vplrWFFsj6a#_d*dSl&;()CmntJY9fbpFn~ES;G8Z-Hi`^(}m54~S zx2BE+4DjJA%#AWGraWS{PnN(iVyUz;#ISk#q9G9HXq`u!{pgSfPgomupZ4twgSc$h zlIO)s9tai2O5n4I6{CZk{m-ziDO>`Z?+!?jl4p#aQ_c^b^_yo8;QQ><{d&3+o{Ci` zoMf9IF&79VStjh;sBK71s;whaxsRB6zZ-*#UuQ|E)lwPOvSy)c7>%ys^_`6Ne%o8kt)2m=cf|_Dtx&4dAh`XXJ%Ttbs$ZG zQ~$xXFRm`u@!Xv?pveHjfQ@gLvIke}80BMn`8EDBF5O^8cAB0mf6a3Mf}KtoEl*U| zzqh~um+)=VzHJ562wKc@VdUZAk&(%dLQA}RKwtL(7}AS?h|_zfJp6uoR&QV**o+@k zQw5Z0W_`Mo;EE|KXM^W%aVl=x(L0H=k)(TnsN*4#DGfWnYgmUN5x7dPfYkJbmMb?9 zUd&H7chOO`WAFpx3G{Jp)EDgnS-ZK(RXBtJ^=gU^n~7%^BO!qW2J?9hKjMSI?kpLZ zR(_lI>-k<>7fPeBn z{B;amAyy`_(S41Bm|T!8Y0ZReCIi%MX}i1i6;vx&m=q9ofHFzikoYe}vc5mBuPpQzGiy&bUMyfw&pXd89QX ztB%S*IJ%;Q7iO)RkV9tsh{<-&(^cXNlWeRfz;_*X8Zoy+twn%8x60fnf`i?7sp@vq zh&(~JOtgB5+EifUY_;L>aRmyAiH<+(Llh%yptSE#VTsg(<`^${^Dn~Bh|1Kfv#{S~ zSc^TdM;()8geo{2de3*gO>?`wesdyJhw&HBO`hY_YIU&{l&*d8h2f0|kpqcGTcsfP z(-}(8W4Cbz-3sGkz9t9qk?|7mVVOj-DmPh}03ALBA3CUup_3^}B;k=y5xb(6!;r0s z#@sl%vaYxuS?JH6MS(bblbWwR^uJtymguEajnaG@d+rvaoZGo-TGl+JYXIi;;m`>^cgc%HIy^ z*D8!qqW7(5)O(4wDQerI^h@T%uns+;b(JG>fhwTk>8!PEuWnmuhhrr8p zV_5h@9C{{jEXC+Lk~>0(Z9nzWFfKQsc!#H$wvKsj*xp9aH}>b3$9uhso0L`BqB&m* z^wv{gH6;jyX*YE;G0~wF5vAzsvZjlWY8-eI!rIoeHIc-#-*FMQH1imk4s$p4vih0L)zY5c)Pf{J5zJXN)$fdg;F z*I8i(sKcwYM}ziEG*5{&&^7?YxT5hVA+~CoFd?!e5}(<0Dw|`EEHT}mnc|O-8Q>-& zX^;cqxsEgegp0z0L`&0lbAjXVb#yHRLTclXi)m%37}37n2`&f!i!+B-JKZPB-EuSN z*7jY6Z*qT#TR{wJ&8b@b&yL!CC^h5b9}0}O@G!z)7fOn_s$I^rVj34i-*g)am>{6v z?z&lC4+*S&`vwM{PovKBR&o2BxN?trehd#TT#A5Cv8dVzq>nNw=)Ii#IO?GojuyoT z%;dS@k(J)&#|kjpSR}b>SUi9?mz-{|NihkOwsmHF_iM{Q1=kt_RUm z_d~3M9R^e_#oPA0{GR?~Cx;AJD!%qwrWR*s5mP1qNoP++gsY!MZ?p zBk{`}`#4;Wf`v7K-v>Oaoj=0UTaGG8Q?I9JRc&Uy;%LXGlf2|%z=pvK!>1`xaTAbB zpkF5}6u}2mmHMkF{UDn|9Or^(%GB=l%AZldcg}A)2iDc<`590$cmFd|=OZAlbyee~ zLT_I~Fn2ZbyVF#-4Y-tQvEkB=k=j{_Bia5l2S-AC*}6wY8qMFz!!pU6C$B?VsccI{ zRn#rd8!1x|h(~ao1Cc=N_42VV!^&)?1*t|{oS9E?BG=-UBI6T*)wro^(i)0?@XN;vTum!&-n_7Z0Y5ckXBeNLnc{bq{;je;0k?(>FTC!}8bch~Jgr zO(JBS=yT&7$hAM}AA9##hUmSG>GJi4#Ak7`FP^6h)Jh0}4_ate0ywI|uMlBqpAf0-etSC{CpT)ze5~tUng#;xL_c?h&NUJ+p#%W|MuqaC zKw3OMawJWbpTZ9++^dQ7sP#o92Qx~)XG-OsXupXz_G?&KYd zQQ}f2ZVGfC@-WQu6}ifPDcqL<{OYj}Gjq!tzNK;uITwhd{3r6CrBol~UT}j5v?u90 zQ#aua0zX#tt_2kcOPPF^O0qKz@wHI{`w``ZCxWiZirCe~z0=e!%(n zJH~jA#k?jf_oU6PhyVbzNR3wD-D(|(E9U#`j24Ya)%%RhGKhL?h1)nbHR|X&TbPZ* zkK;`Gu~Bbhg5sHu$A0ZC^ai(&TJ58wV{E<4s~~T0%oLB(%3soXAjL1I*o*FQ;h36S z)IzY-iV#h3aii0{6}vy-AoSf4?Y1trc`U578V0L?Bn)GqxI=iN;P34Z%kStp`N+~Q zsqPAR1TPrKR0GsnlXDp(KZ5ZjYI@L}LPHWfbQa@U8eTTwaV@bSiG>&4vCMd!-~751 zjm~rQwHLmYDP&&(J%%BBIsD9gliWGSFt?&;{`7HIws4>&f$q&=++jjqc9$J3`Nm36)i?qf6__u&76`>UUcn}BmF|upAoGEc9k8E;Z3>RvaX!&=! zN;Y_6fBAfF+PE-wHL3!DE^L!FpTbX$N1tdFK9*dHCD&$e8%W|d{~GM(y2+4=^%;CX zxA;F8U1mS4^*K|%v~&k1VV1AtyrSp!9G$As+PldLuU>5lnn-v}q02JbGUU})->u)w4I0cxtgtN#IrtU+D;P_ z+TMQ-f1kTAdYn|M56=*ZJ3ga8BC4_XOjbT7ml*AZYkXFkV+yHwX~rJ!v4P6cdM5Bp z&mK-De(T>-R^OK4hT9%M)~lx7hqaUIv0l8V2H<(G5w~2Lq!-j``ZT^Ygra)>76%T~h>rOZDI0TfX8I z8xK-Y$bJ6%S?K0*GCSSELIzhuz(O+{kx7e&BgylP-XY1V7k$4rl{ws1KXQS)mSLa> zJ0?RTrIM1uzAxo&LW3kawg*k!tna|`eIAq9cUx!4V7?dqmMgt_F%G)Zgy`tOsE{lp z!CYtS7i&fLzJ*5Oy;+=P_q!R5?x)Cpq3)jH@+w6Lh4su?;}>?ma`F>@)lj$!JcWCP zA~o;T$xBzqdZ^P`?c$lb>tTodal?`3cw0gK%n5D@&q;K>%k{n!zzyYBO$k9uDDH&j z)(^r$OZ*2t*If&*)=(6Lj;>s%+PAoC_pUgizUc}0b$q0-*WX{&F4C_Gc}i6VfkG2C z|H^z`?onR|lcsChDe*rTfR?4)-p{eJ{P2W?Xfs3Ub5`-rbQ7g}VPL_}dsFvI1Ss(B zO(6{qgjqe?KwNR;EuxJ`@`b1$O%w%?V=d+yI!qo1B-R}e4+KrzxJC{QoE7rElGwSjSOBej!Clm1; zr3J{zN_l(F>Q=L+$hcMFa#mNJqi1*C6!(C!zoS1kz_3G(+WZC9liU~<)A z0A9|{&Vnh%{jb*e5AQ$R_Bi|G)GS)S!ff8>#f{e6d7~aDUA6Q*{#!-)OCGOFbW*G% z5X^ZZsF>|rq+~Bm?*cKmc9lQ3shs}mAf2jEYGCj@^0lEG1DRTYZY%E66@95JV7lZr zq5vhKC^GR3*$oc7KfCeluRl7}%m1-y$ZxOOL$&ccqQ&2vi36odN~wkZQ7Sl5Rq%Vg z-r{}o`g&47yY_n7;3Gwfz$ryuVSe_HRx1dh{dOXrqm&`bMCV(~Tq}j;#1)ZG0GyaH z`g((dtSj>yTUp0JgY8{XS&YDm_lMgn6K5*;%T!O7K!}<{UIEkc74~O_gRce;9`f7+ z#b9{L9PJe$4by~KA_1Kt7;HKfeD@#pWhNODNYv_%IB2#id4c1hIphCS_myE$MQzt~ zcSs`xBi$t-H8j$Vq;!jPiF9{}G($;>gh;1=bc0GGEg>LCiQgWc=l%Eney{80-?^AM zv-6z$-fP`!vAbr%A0KVJC{R@OgbpMEHS5)%Hy%+T|;jSaO~U?O;W+7=BCpN_(@2kKWY{Gd+|ninV8U8T3KzWr98KPDjO1vB$OXV!LtEml-L)4-l1Z; zUC*)!eSH6ECCTYM*V6KE4ATwo;m9Wco5YM6H;c2trI&+ryEE^S zwno6-fo$4MU2Wkpw>E&t>46#k5VCfPsqmqrWjNVg=qi{5AFb}>s_Z6ro|C1QV98kz9$X`C+urDbsE1YwgfOH1BEaXwtf8GjDv!Fiy z`qh#>3?!n);7Rv@Y5cKldv*2VHNmXCcJuG$Z{LgzZx*9Oe4O7g7A+nHsbuYrCKqG+ zY+S-6n$vyhhm>QK)Rg&)X&iJAXS|_A;fz{S(Xom0{gqytHR@L35%Ru*gh9yG&dnn+ z*!D|)MC|iI3tIMD@t>029J~CB0RIX%X~66&psPg4zfV4&o}Qi!HCI=Wx#_GWzn-Ex zY^K>!>&{|YBr+}%^j|(}g!{-v)aS zznCo7Vj?#iHD-ivqHh}1JS$ydi0e0iEiRg~dup?H7TpERdD={rGRq}IXQv=m?Ii`u zqA+J)kx^Po8mi#FAW_EtM*%Msw#*4O0TW*LbgK7qF(hRd2AV%$FtgAga2f^7wBA8P zH>l>L2ymBr=w9q6F)w(XPgj>1Bi?D4$U!M4Vt)dh%^HZF6KmP8{FP#TzClMc_$Do{ zG(P~EFCH89Fo-fwoTeb{3msl~_>#;>cRi4m9t5X|(SXQ7mCAD0yN7D{IDmFqCrz`< zL43*WCj%NfCXI-t1A>?oFSqvlMAne^>}js(OGCddJ|S6atv{S9m_S-1DN8Hr9j=k!q=-Z}$QEMlTAlGvQOHQDOH zGwV3nd+6(hp`nuvh8dX#jk0ueB4S3GlvW6?ok3Gs{uNFyCbWxu^&8G*2%=ly@>q;g zsUYXOtpo?oJ2rqS2C+WfiD&EI)Vjr(JxU~|mr@!TAO76q5?ANb=gdo{7+8U0zg+ss zok~dDpnXVF`7AWpoJqR(Tc?942&p1xH%FqQxf5G!t_xu``fqMsne!z=Go+Galz4S1 z>u|`~FVXU0Fv$;i&sw7_A5|5)6s)_xL-7%e82zn#@a-23fvai!K{Z}Fr!lrm>u;HJ zF||O80e&=@wS5_ey;J#vuZ>{3Zvi{_-Q2OyIrHNp+nF39%a9A%k~Go@%YKa)3<6wK zqc7(H(W<}*2?BX&6bl?1DLD+@%YEA)n>-kRM7|PGQELr0b$UH8Wm=W~1EyL>Qm)@d zT)E&HA3L`X;as}eZ%*P2!ILH8#Y5iyxnXteLh}#Fv^HarCiqmyxX3I=ToWob{-M)E z=(E7uM8OK$Oz&kVQ|Mpch;Wmd!iWNaCfXnBdM>l%_}^D->?f9@SqPoAQyyG?(MuZ^ zqAql-QD8N$6Jum5Cf0&dYw>M|$3FIiO}}MWL&dT#c~{Zh<%uHp(w9e_b~#FCkYH{0 zB{}^X<)MdRj+bw#MVn8ZwSNy%jnWiTbo27o)VsF zuID;Q>{a`-cu6?nf&XNlWHZ&^pPIp}K4UuJiQqVSC6yGd2m&d~-_4~GGTn<zOQcOhD^f6!F zipa^4E86=KlW`bf%RkkXn`(BC<~AFs(y`j7DBD+ms;s+bgn>_O{WfNGHwGJz-L{;r*F&y@-V>0iX2qL>mFlfd%mKysk}~9jKkDPt3)s< zNl;iWY6i<93YbK&*49@l^*GY}`QHQV@5cwh(o$f(W(DYo`&z~sPOXJOIpVlk@d>oq|kQSSZYl$-Wo7!sGw&K*U_-{3U+cytUD8%0QLYwX-ZDyjpHCtW#S@hYvWb zRhxA@%i3}L*sO;xg^!bZ=|86^=K-qi^XU?0k)YdV{wclcMg9Jg*myoE+IEUE4pSTy z11^+{O&wq)jYlg&Av$KyG z>*mW?);487W<$E%oF}MrCF<$Go3GrYh5!2EFb=>K=y)P97;=4jpuuOP^0S?ZLBNum zt&Z)_RBWN<((0V|dNWv!Lu6<}9U8oal`Gj!KKQ4)e+ZQ(B_rkiHIH#qb?o2yRxhW5 zgg{Nh!tijMxLm@8;-`P=ZAQ{rOaXb0+QGSkJ(|ZHv#zwN(06YirXtxM#NSwj85j$$#y<^gLPCQiHNWGT56}C8(dCs`Eix~LKLcmxqA$@yXM(Ve2`WmV zCS5%;E-mD#{-+!o;SqPY`WVC9Yewze)QWe{K@&m5=gzakBId34EL zVkTK&oJdXh4>0El0C&hh4#7qt+vRW;kaAQ4lqCW06Mp0 zba-<9=Ne$r0rwvRL6Q@|js5Wb_Y4tVLq-7X7r`J7KzwfjQHDYNd%Etw*U8BT-~;!c zs|D2zn?k##R{y<*K)#`ZA3}W33_#OgoF<PmU zf{)mycWtRDDIp#^;|R_x@qK*fix~o*)8#y&pzxC11wbw0Ig>OUWu>L-vlTk5T19A< zmX_^&XnQ(fx{zqaz46x9*Q4X&bZkXaIIyCb$=1X}pq?9_vC}z?w3S%97oL^vEMn0D zKXnJ#$;k-}R)l?ZVJqYgnEeVxY%bU+5sF|n@Oc+ulLu*B$pembmmxFNrd}|D^DwFR zVeCvpOKG>##5=2z@SZBM8MZim3jvh=q`Kb>SiARx52Xo7>}CM(B7Z93he*laGaocQ zy|?RoKyr#FRl$Wt8F5dc&tX1-EhlJt2EMacdIZx>Jua5uMD9!O?P+xm7p7! z%w-ecYKtJ=o4%RR33dYhj1mZ;DoRoJl6-c*VZiXl!lmHW4H2MfX#p}e&3`WziHV8V zy^XbdJns1@PgdoFzJ}JOOs;!;`|2kB5L>8%B-XJsP?ogSbB_eL^}TIFeM{>G#2MJd z{7O|a++VMBz6yA9HYyN#kPfJ2X_SD?TFihgWIu@nUTL~1vkTQmVSr5h14Q?l$S!Ok z4;;u9_4ozU%qm)%nW6~jdVxvlOF(*tgOzxs+rd790|08%44TyirCm#Rf;WewFn#Zy z?dCO$Lem<}&lKnQB=xILRLMY2F8lj%kpdK0@Y%coG=|QbBk}grVGdtV)_npj5_b|^ zfc6L_uLY88`-wsz&_vkZLAdM!MG#YP@o@NHC(#*L#X9I8x1SLlHUA*+sWi*5LCpU9 z;e=x_n_1}9D>*R&nK;bw=3R!y3i4RDhhB5u@-S8AX3stCTfjnPk5B-EA)}Mm;k?{R z4zhlEe}6HAn>X-Q-n3ozQ~<_gm`eWYM!f_; zvtGe$=%pR7@r0Q z%ZDQ`=;O<08LrNTibRDf?SeVjr{7uY2`ksDdNb@CaQ)i>5710CB(JQGT05?`b1s1q zx)vOUo^fyhaMajpW{v#nBNy6~NkCB!16NKL7UcFEf0gFg`Q-P!fSZ8#r!a>5P2pF1VRQ240 z%k6$1fF(@{ICIW#*1 z{SLm?p#pN$XP}>v9}x9<0^yfbOHsfG}AVKi)X)mE7Jg+ms@V4YNKS&dg z0q%2>S%?4Dn;c;mYCu~m5@a+cDGEa#bM`+jEKf+`+#b)vZS~h#+@EHzG1(iAe~a*T~P;rYa=<7j0y@t-k%722i|9YfWfnvkVMV-Oj3a!_!1%OvYPS z`b|g9M zrnYY8jbES>GG{mWQpin+jX5PO+k=Ox?ajsEFm949mm{-(gl)Iq*>*($^$O}#iGoLg zLh?8*>F(b_aW4PayeEFz%SI>aQRmT&3H*903qdAMSmcG{Gk%2|52?xJvdOth)_|iAXRPhT!j8x$ zT5fy|is$FT0XE(Rc2X@h>G2E0d>=`KLS<1=5gr6`ap-s#kp(Qt5G3Ezt-lW6zs57e z_nZXkEQ}*(PQ1%$B9UEww=@DOX_xbt&r4(DgVS|ptVgbNwm41bDMV@xv(J>ovgdo- z7&-;qH#xhJr%?}qf>8h0I;#w0q2($e636M1m}xVh1u413!0WNCs#~Q$w%+dhPp-if zv^?D-yyFQ_^cv0`c&z^WyN4VsJMIr4Z^wYtbxXLX?NMR_gIe@6b5Qa33pi)3MNpW1 zsrmVZugBXtar?z^uU{Rg?SnvgT&0e^Tm-Qb3JM}!6!QjO6=NTKqrc{kdh!UKL{Vb( zlX!>F{AUJX&*agzoQ#(OOwf30w%&i0^$w|` zh1?D+6sCuh&x8Mn1LV2kwA=&gnbGNcQpCSJ+J)6Q~>ekWVAp%Oy5)c=fxkvp(^ zNSxOH!<2kZzWbAdJ?a{XgUyLu#^n|dyDdjGOe9B2PhAG16Nw}y3YSjeqB6OOk3cU; z+yukXv|7U@#>PRtcrWg=JmHf?Ug_D&U9oXiLQqtN`jI=8@FRER645kL);omE*BfZq~)u2ZTY6 zf?xoq6xd8#8yRs^<#;7TkV?wR*tsH0hN5Z$*Hj=zziNo3-YKVK)0uWr#UURu zy-{2@rHqLlr-bUadg*{ewJ>AX7I_DsrsHsvvDIavlk-zVY}eVTs76&V1^sLb%YIZ>!TdFa(|B|sadx)0L=Z3Ui*5(y}MNjtH#Ms~W`N=mm_WoBa0Y}iFCtDCTjR8j|Kt8`duk$g*Rhtbp=uuquoLCaV5k zlVHPpOK5`wD#wC?!8uoj?jRyi zDtjXaZ_T5-b$MmoZa|0;jL_55Z8A=|#rk!HB&&T@36jze{GBjZ1(m4b zM;2U))MY3fMBp`aMJOe)SO8#0LhU`pCofjFKW@DBlT=8|n;^Y3{B8hH^{|W+auO9) z#GMuGWtPz@$Ay3XridRt1%;FBXq)O^*h>#ZX zCnu)%hfbBJj%QZ?8}-{cads9-j?fp9o%*Cl$vzmTu-W6CTxli;QkuB6fUh5ulQEI} zz_`Z#8lt{R)bP`?fwNkWrOO zZgvg$z*ACr4$@qsq!uj+9BjYf5+i7OERyA~k5t zd4i_HNQvimyz6Nm`|6MerT(OgG^gs z=$;`Y^n&(2u z^qu;DQ!Vkp;n(i2*v_4xxHcK{rv(%{#Y6viM0c+_PjpP+Y`+?x>P!7PUfE)2n;NCq z0-^j+k%1C~NdR*vEw~}m<9)J{X7Y5R3OOz;W|x5%NBEHd&E<`uaM}Bz?tO_Y3nV=p z)U=gZ5i&g$^bj;Vo?ERw%N~0di_hq`!Zz#}Wh|`bo$dRne%_$|4;SO7^?**9)Fj6d z{VkMQp~*Hj`s-Iia<&{l|8ixJ4oQ2pxhx8Ah4kwZjy^3hSfYzt@h)BJ0yBSg>b5@ojK;(pYY2KRY{}?8S*gz&Y4R{+LyKN@qoA;2#K&&Es&r2z2x9CY zO1Iy7Cp=_PhF9$vZV&w|=Q>(kj9siLaJC1~R=rk?MW`40`Ol-Ci6tqhcv)CsGEo-B zEuyqBVOp)m9pG`5v_mSj93R@O_)sRQYVaYDf2J=X5W@L!R+1u&dS69m?$T!OI$P_5 zhRk5&?8u^L4pydBd;4RGwL3gJ4BCpFt&Hn3yDPl2L@XF~(9c`n>Z~w3@2;KXi2~0M zvY3E;^l0K0p>p^)%{HIo_xkCWQXeTLXKz3RpDvnk<@*+zXe(*FZdcoNCkC`4i?w}p z%wC=A>D_@qWL)Qs)pKs9)JHRXloAgJn||n->H(FIOmncaKMKT#`%*Nz8ArC>`aDu~ zRpjcg#w=soGl*u@!i1cC%su?rlao2nO>6 zpT8DFJtc_FT!ucmtMf{B1DQl(f4oo!HK-pyT?GZ|h}FXgA$Tq{!wYRz@6)Wb=lwp% zCw=b5|AHFrDmX*=#02}{m!?{mQA1FdnMZhR$?B;8>(d zzJW%^b>>H+`^kp9O)HIA z1iSCw5yAb!Q6kweov6>dTD|*9z{3eu%G!vP_vf{z)1$hx`?i`@m^GO-DUoBv<~W%H zNeC}@Np@y!Z+m|2)GFtsj+9S(3(cS~^F;<$mZ+`Xhn3fel4i!YAjhniwpNk`@)0`O zx*6K-BR`Vr~ znaq5)nXBhB@XZke_Gg1|0Wi0|sP^k(7 ziPJ1iB{-g^p%eEW{gI+b%FuCAe$FctTr*lAWll^Xao&8VkRU&itwc2yj`ETa(S%Bc zG@?~ZK3Gp(xN`;#tLUMMNM3!N%zk$gcQ59%yd-DBirtw1`u|0=1*3T4bG3*VlV2Tc(3I4{;=pSOHak0u$S`w*r64Q_;Ysb3n>%4GF*l5g@N-XB2Sf z>}+Pf6J&-!UO%g8w<1R@)WD2zZf4XU+wazWQTqPfiuZts#JAdsa`-d#S@$gJ?x~(v zR5+plv50DNDj0UEtgfySCkKuu4{Q#r_ zgR1Ws-sk($B_6UMIO9iCs1*cYJn})O66`0T^^Yp4=@bW=ty$N^KOd8%bDnkDrJh-^ zmF(-?Z!T8=xq_!Mh4SvN7UyL>W0EEVBC@@aTW~sJ@elpY`_B zBc{hfVZ!od_G_p!cQvNg9cM_S73^5Ps34SkQRu2#0E5x5?N&k4`wcx8`T1^Y<4qD+ zMJSu$GOYf>_(FnId6>VH^<=tv!QJt~rcWb*kVf#5fwEGLwl=k{ptD);pu*oA#-%W} zr{Gg8;GFVw$Yx526#}G6M7cMALN|M6ODY-G{;SKhMFq~IFr8RHuYI-khEN@$xY&M3`MTt2c!*F-nwQH}1q5o{pqoekk4pdI`~s&P3WRYr0H_wGi(k0IbiONCX|uLg==kN}qm za}CimX++P^W;V(|&yY;Oy?=rBS}>r%U>KneFLl5zk+}cA&DGG*7zBB~?!KMvAsK>j zbIkC;=hzGgpfhrFtCt2+oIC`#eqzXB`}4``yC;`ioIuwdT6*9-NTnvRs{k4w8jOne zc1suvm3xH^y-MdWAQ%lADTuiw1V2t+O^(eXDoO=%&StmuP)5c()sgReGBaS*h|o+J zq)CeA5s;5kbhf6PoLtD>cXZuV1lrOk1Q$k>%gO>-3Yawg2HPAuvJVj!0fBC0tb5S#LRS$%`jMiC64p9H2q%uhhgnC yBFb6K9*Vxld$1_M(4g3|*)ZNCJpYS7d-w3eXA19L_dz%k_*0fwmwOMh2>pMO5%y*P literal 0 HcmV?d00001 diff --git a/5-appinfra/modules/htc-infra/agent/docs/pubsub_push_pattern.png b/5-appinfra/modules/htc-infra/agent/docs/pubsub_push_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..68734e12b50cdf05ba7ccb903d52579c7849cc0a GIT binary patch literal 45041 zcmd43XEwR7yxXzh5d!M!U+Iz2c-}e#)(@`S<(Sa~9Fi0M&E9+xmU>0Iv zU_lA+fIBm_Z1KP!OgDWsMU2wl3>z32EEtcK72w_$+Zp)Y@Xqlc^RrHL91oxbP3%Ly z{A`RB7}@Bci^=k!3u@o8H%+H}hF=H?xX|kF=Y02?aMbzJ_I&VP8@InY+1+hw5Z|WF zlz|EfpU!yg&RPEMO_i)}Y;2TKC&yLB)Pquz;XW!1(uUd&CX#>q?~M;wSP&nw|M|2e zD5!hrJNu6)tn45z77zjY4=ve$9xbl~zNkqWPs0g%^Y25{yBBv20#CLM0bhJ1D`S%m zJaYG8kVfWzdQ-;KmpXj*i?v@S2WcQAH7EL%kubv~Qnw#j&!oV&02nmr*HSFegGk65zUWPM7ik zzj`B`+rX#<6d9Z?bToC~b3{fn$Jj1VFzd-QOBBPoh6`K$) zvP{taw_3L~9e5#}P*6~Ckvjx;&jjdSVKH*KBZ>k97j+mZ|9^b-Y1ch?so4{#ZI&AJ z?ls9^Pprr~aP%->!IW5#Em_cAH&H>|SfMNr? zCAOhXA4v4RD$HZ)Mx~Gu<38Brr-57skYFb5eWIOf2cf!qF+V!>bXHJudOzUn@2r{( zB9M00J0~!Mx7|xy^x|WT#v0QH%qTe+C?*FCF;gTd+-9&^tY%=4-MJa@#436&DNSb> z7i4Z{8_cPyC7_9kne?O^Z>%f`B5+M?&*%UvnOp9tR;ml1BeEVXe2xR*P`?NbQFIAi z-<@q3%u>Ra{5g`KtGq5n4hz@0Y%hI;`r29>b|3yk(kG=EqE_I;G1ypN)l^6- zcZLy3%FK6UsPTa-IFbqZb9!g<=A5UjF+^AH9EZoKRN2fdz0^~S^I%AmKf@rI+rpdc zAUD|}`f#OZYf}wETLdgFW7)Txzykd64)cgY1D2uQ_kEj0vc8y~`h}52Qc@_-Zp&@J z_EZ(mZGhP8x#GCa0;4jEo70r`#UB2a5s!W(Ur~k4?6pIs^wfv<6{%UmuVE&S%{BRc_Hq!TF!`M#i0yl_tzcubgYLV zkmm!(Rp^W+qUP%$5uXWCWA>-?nsHGJFwHEL_rGx8o^P?#CLbCLq0N%aRc{qUsDy*S z12wNLi;XL=^z`)XrmIUfxBBI8m7hI(Amz22KJVq}DVC>`5iRAhiL(A4aX!EH?bijp z_$%cC{oH{ZHBxx7@j?0}w-&Pll$Xip4|(o)!1PEeUXwq*iF1`CCtU_muv6!;xh^zU zm(##8oDLp)Vp_{fJt>)K-FXeq;P*rse<;wWne(wmE!7V>mNn&lg(Pzu;F(rC>9Nqs z2l(C4kz;d|w%j0bg%n@x%{47(UiXR5D}R>r|Imc1;ITfyYUb7_YPGs*8F1dYgEX*4 znY08n9m+9zFJKZg%bA&nGErqEB_?LMa*&s=bSHf$mA^{(?DY4~ulL=Cis1qk(>pA+ zcKC~bJE<+v&B)`;aGTC(>!psUbL8I}3DK^NA6?9WX8yN76WMEz$4o1%zvFrAHS*70 zMluJWt@@UVXAm^xXCgt#=ANnJ)e-Wwe05pZtH zfxQTge>PQV53)IOn0S79va@-qWwZ-|H2>TXC8Gnj&*Kz=Zvrsc%k6JxC)2ONVe7c$ zY_knsMnM;wMP+2n%w-mbMwb3nAu-i+V#p&qzl)80@8!4RBeLaW^~4OU#(ub?+Tyce zz#h5`*ou?C>%CVq;~Gi|3W z@sks2_-$;JMRGLiCjtG?*F#vDKF-u=95gB=yDqq!0v*RFoo7+&7yR+j6QMw-viePk zM23!5s*s9*yg*NHZ^vHCZK>4{#rPL}pG3R27Dv+X3hUg~Y$nRB9isQwJ_j25`$Fi@ zG_bY+<1<;EOPBgy4~_kXz2@)3y7F%cDIRhO*oV(80InW}C^9VYwHx|ENGD>4!dNqz zs&b6HINrL%DJ(Iq$$rtF>3y~s>9UsLwd&R7`=C|sa$DGaYfL*?E)$xoW;!hA&Ll+z z!!>pBAE$tA4F4K22@Z+27$_jN>OORGl1YfMizH#(k{W3C#K~s&7rrjpc#=}jCfzbf zG~R*oXS^Pys9bp_1!qpX_lNCWwb>tNq*U|tQrgO{BemFZ7q=kZlSd|%1o)&(@Zo1T zxU!rfQQ@uD9dzTbB+N%Eo~l)iwcCr9*rArDi1ODZcqhs%TNL}op`BIiyk9bgk=)$e z4lg%>`Hg&=HjW{Hm5{J>YF5@@Km5>EKCAl}{gKynDbshi4e(QJywO^hWy&uEdPrB3 zEF%>%xq}W02NPKQouM@)>;5 zFDIR`#^Qds5XX_PQgDdF_W;gjUMr&cnyPnf&eI%;>RhGsRO}HyDA4O1B#csUAJrpb zhq2Pjk@gP&>&wkxM_D525#WNc@D$Zc5*j3n}*xa*oazxh>!Ft6OhmHu zrJ{hV?KZwqF3}B1_U$waWv%=S#sP;y398#exYyeT-(OyAmGwv6d*n1DvtdAkZnUQ_ zUQ=#}3xZ5UZ#85FX?)hn8v}LZeEP;sJ>TDdXlI#v-;;QcHY)nyVx5ctz51%V(<^*J zzc7X5)IDCY>g$D*prSjlfy{BJmJMnSDG-MJp7ol~czrLFNaFZ- ztE{C^y8$=nhJXwC;3&HCI6vfci4blk%`Q=w*HIuyg%v*RbChl--s@Vp-x`dK*2{?~ z>?(My!zk3PW2*bqSuIWM<%oMOm6i*zdxX$u0mC+bwbA)&S^V9;A_T?8KtPH2!1)?D?;Y0O`daG+;>=9HZYhCw zH|gVWh4Ro`DOkC5CmJM_HD44S-rSrmlj1(K*lNDqH5iVZ5$QcHKAjJ=+zEAkdy%uI zmL{q4^M;V4{f$L-k=hSmCgJC?(XM|)n446PzVA78fAppbkNR&6MvC`aVywFy)F<&6 z89F*3r}6%>Lr|c+sf#iW)KFQHy(l*3=nrh2M%m?F3tF%6{KqrnJfK{UQW9LO!sE+y z@3m8{c&Q{_`|r0G$~ncPj?aUmEsOkvBY(K-N03)5QnL5r=*x@{82giff{rR|Lw0I?{@0eCYQ>m1<)l* zFX^iFvrfOiXVOUK^Vdm1D$t*2uUgm4&=j98JQ3bC_!jE;bVhfjbiWc`RnV1P`1=@9 zsh<5y3GJWSO-C$5AtP0m4$5UQP)!XDS6^+}ivh=6{GZrfRRxbd^YTk*P3pIRh~!bO zKd|Du#hbJW7`f0oFVZD3ulGpmoNv5{ksJ3t&Bh*J?QBWF50zer-%KPL=y>eTXZ?{G zeJCwXs$C>GFc#?#=ca9a&GSt#t@;XINA=?P&pm*SyB+nhDA|md>eGw9*ahm`_A31`1KVcPG749sWs0Md`Hf4oLAiQKc zWxrI))g@RIrE6!v$$R%*19u-!r59m@^3gQ5`ajhN83D0oAE)xafD?>P*Rk4&GuxKd zllUSM#kMV$6kn*=*ev^mD#1{T4A&1xAy(d-DHJ_Ifj*67@>4=!2k4yhG}nYDKMOW+ zOI9t=rN<>;GV5SgN(w$QAO{>{P#;$4d>bZXYZCne*P8&M6hWKYsh!vcG3z&}y43nv z#h+^$s@~CgF{HQ|jisUlq#dGNzuaLid-&EIoefwrvO(OmA&9Zd~1hjw1q{IYLlcKj;%+(Imy?axkSWc~ZYPs_+Xyw4ty zhOV3kgs&tEEt$++pcvEH#)bmDib#T7A1zmFj#Ut3R+p7d?FTSg}$y&L#Q`)#G(0U3ayEd!r}3gE8f6n6uMg$a(AU;1@lO?Qp; zw8ibF(lpw2;=`%%D;Ke$Z@(ioFJl;Lv-}@yeyCYKv*%3VPB92A<*aa1jX%$E;G9%2 z-jUwzIOC8^se6e31oy-e9ACY7adM9`raCw+s;WJ?>U(D7W)wNvXs~o^yA2uWb3V1byut$G9-cvDd0% zf>WqTb-dKvWl)7W-*hMrEZ!`&T3LBNG$LX1$@@>8_HU7xcPJ<7rufDhGes5S}Qt;*7BRVt?{pa8GIkhGxCU(DX z`twfH-S_KFYy&7FFsg zAxCpQ{h{N@q5Q`WY}npbFF6HfTJ-jYcYHqHo;0v+TbUwp)2n;+byYKkdm6wxN@dtc z;99!#)h|s&b8DO?g_8E{JKm6=+izDD36Dxy&S)+FZb$ZxnKpR7Ogc_J&zb?tR+GVx z22!MBs-1UYSpG;+E2y*7@~CcBxGbysov$Q2Tt`_<{mziC0ODygk5PnG^)nJUv5fVF z&sf&!&qu#{*L$>g4EeAij&Z=5hPAvzjXn$-^>sy>Ir1gW zC&M$g7ZIkV*Mcdbb1I>DB)>6Zm`n*EQ~g?XCWorXAT4vt3Ce6~*m)8DORFQ+P|eEr zJ@QW@ir8S;M(hS%^g4tX{ZCCD&NEz-EF=%n}HoHj9D|0BCbJg z_ns#)s-+uO@lac6>75r}h`bdMl?Pk)`jI=%5gq4>cRs34f0!fQo{fJma}1qgkvW&_ zh{L3i^6_KdP%-tRM!9?(+WNi?rI1()bzl*AOfT8L-K-k7Fqr#{7ZEZprcatzb-Nrd zKhWUydW8|(E3^%K4qQjfzUpH1nz0^Zw^MJyf+#%`X{5kRT6}jgm(_xj`uHv~u;vdH zouQ6ncjx5bBAc5qwX1_m+eZelv7WKDv#Q$rmy0n^Or^7U)9IFpk}N2P3mk!bqc2^O z;@k5+{mdk)kejQ&{Y7sSb$q~v_ZTFCxDE2q+yImi!QHPF&zlKVAsE0rSt+F{z-Q<3 znZLccESthgNJ&ADO-Sr?y~F?xA^F>(_6A@l*`rIf_-+Xsd~edwAGH&iax&YLDySz` zSxTFAf=GSheg5_`W5N$bX3F-`&O)|*gj;7hK5sy(;D*q3SIuS-KDk0yZJ;C1tP9_Y$BBf`DOi2ZorV#S?l z{B*3;oJiGRNcf8yCraqA0adgA9cFuS6B3sR$E{j&rm+ zvM`p&m2ep00*C)(A@}N3m!)y&zjP-lxLrxM{9vvte_a{Wy-oTg<%AzNCR_2bAbnRy zj~1X`DY#Q*yhuShE1KucbsvKyKeih{f|dqf=+y344Sc-*rSrvBF+*9w^xcxvreCWS zWfT7@5Q8u0e-6-hwjV*0$1zBTJbwSMAo-;N%*BmQsg#gY$M;|>FYT4S0=dldsExVV zaMa#ZmF7-zp?+?32;uji2KxGN2`S6n^wJ_k5!KNIV8T*DzKMWCVmSaC&QlUF`yUNn zVZ?;c(HC?9pcQI6?-X8QA_|_<_161r`l?O^3Xa8>AueMDOhM6OLfBu0=Ii@O^Aq{T zNh<8U@y(_~R2PN5c$yPitLS_&aA~uOB#KvKdth7}(1{U@OGK5955bdjc8K?_SnqS+ z7{YBr9_#Pq>3%M4uiUHO;;Ru?PjJo=%vq^4kcYt0D2xKejW1nPA<>Jw#OVu}eAU(y zGO|Kg!v=!-q;t1ngcNWXL0NyhFN%cNBC()lL1uZ;6AT2usxu(e((2-PzAc;R->c42 zLs52TxBQCC-(9RvZNhllc9pVkLP~0zw+PWk8_yg{V4=}- z!(M;CAx1JA+6y|Q8`N?hgWclggb>dtq&bUYBf_IrHO9la0IFqm`ECGf!|32zn1??q zwYFt8ef|o3pMJe<9JYG?i#xkDy!fu+9G<%uHO7R0 z2(Cj1n5KsI;kzX!W#oUo72;4|t0%V@_A-X53Mx<%7z5#J>-|qd!9m@WPyDw=AgJuo zbi8M($Qx}^h~o20KYFbK^Dkh4>u@{!w;%HacVoG}0VtWDKm-u3b)m@r57-AtDggq3 zRl&Hkzv~}deMc+Y@gu;+Y7YV_J1;bxF<#sKpe!c(8;0z4aZA3RazNPT#C2+LB$RaR5OpU0@nHoar9 zyamqMtRP41YHKqeplDlMx+Lsx1?C< zMFV{4xN}ObB1yB~htq%{_WCr`8{D>A@(S zOc{Ifo(^I?7C%ZZgFIqjw+R6P!yJnyBpL{c&t@9K_ux=)c3W0{rQIM~v3Ubk*aycb zK0^@4QcsSz-@A)w9&^Gtw9T70u@Rw`dKGVC`KZ`>R=#FK2jMKzJd8fS&Hg-p!Nw_# zEjgL`Q$x3HWBD=xdB~!#!=OC%OX+8~dS8nI(i+DQ2)XV6n2xS4>b^AoB|{=~h(wUI z=XUEm4$Zre_l}=2m#j8^+`(a}R^Td1{uz!<__%7}O*COP!j&6v0A~b?@DAlW(wALP zO4A9h?wmRdbK%9k?ucB^ohgv<%c)9$8plpygG^6r=sb<%|ej#cnsZ+*ZQQFh$W*N*e6ffcWp&C;b@ z)M+hiUYQfq2tb21qa}ff(HijhyV`fQ{jDDk!_hq7)P{X~^7_GtvX||nUo-*J2&%Et zxg{Ijz@EjQf}i477xakR4Pe}!2Hr+SgL&|#(}`}7SP<+;2}w!X`M_J*RpN0Lf&bA0 z^nWJx4Z}|2bBHnNyFhstxZ+u;)ZYhOo`>6cX+aFPI8@rOVEHy7s_!E!^^yH@5v6xMo)W8M+qokDZdB!=`dqc+=zpJFE zh!sIh8v=|r7N@5PV#p^U(O(NL21S|W_0xzDGYPHC%1KgET1((LV z;_0ZClZ6m%o#Yd{xkgFVSQ^$jL{KO<3IuHxbsSp&U|K}J>OhRcoLkEFYYjE5Vw167eo0M}klK4JM6J7Wdpsy0ri3-wJ(muA>t{_rQ%_-JeEr@C|$kU10yc&7DH#p$1U^kYFXzPd0v9IX#R>_YM% zlj)Pkab=X`egeiOtO09-yQ{+aaBJZL5qXc%86aoafS(!LL62K32%`|WK7ZprmgnBw z+b^5igJ1h)rA!`!|80c75iC6HxG{N+;Rb46IZwVl_g>G6RAyrY>}!1UAtU20wWS|9 zz#=NF#7Lul<%(}5APWD%!xGl_HOA3 zaOSe=b^#$_^yIe}InPqN*98^?%N+H6bT|*sB7?f6|BBxdRQ_S-0#UN!nl0kcJ9##M z{1%vRvK=VPzX}fv1h#X1dC1%jBrGuj*B4;c#1O~ApzhE)tsX(oZDZxmA6b8^bVGj; z%YsR?e#>w&cydaO^5Ch3Q4RiOMu$k3x{3lLBcP0#_K7C-n;iIwBwgk2pq@AH5}*`eSnaI>|{i8FTO)FU6lH z6nBoK9t;Ms|CKmmWeGOx8Z|Ed9&~$eJJG|q5JsKjxy8p+`o9rSC}JT>!v8`m_(XHwJO5AQI045gdOerRX8yHUNAz>*{6|USa*;8OeuWQ@Buw`SHRS7P<IaX%EWn&?k%)o43 ziefz9)%9CqMr!cUhygmqs;l)CHLe(B$OKq5k8)c>Mtv*1mmhaf#VfgMRa|Yz=e*UD z_Mp7II(wk;@lmk&ydOnHMTM$mw)zPt3M=9L`%r+QW$-=S?d*J6rdsm6$%8Zx3!y{x zRY*ADBl3RI(#zUBHQCvn+-zV>j4gxDc}}ece>}r2Oqp16D2Y}8Kc`&%I`!2Zggo7S zOaHoYFjkYYP7W82k*Ad!!W?)--nsmKz-PcsPBw@RT`QNAiM$V9epy?ce~J;ycKYQ_ zTQ-!2N8oiQ-=nF7SR0OIatZ_DIJDj8If`cWt|Jz=03=;Oeq5n}i^JH5+^}+d#PjEQ z-L1d4T28Q;loa2+CJpC759zd%SFDqJh=ip}BGGjYM>EtUSdz z^~&?jy%uMW95L*&lw--0*H6B%VbBuN{N^ZDPHFVkd2glj?T@-E;noP~9ZF5?$4Mz>#F|A`WlL zQje*q{2#>%oP{_eujZ&drq8^7e+(+*lE1NO;6mvJ`(K^}Zv@=Ge;?N9?E-|EzP%jB zqrIU(0D;D^oss0)wgus8@b6?yJcukvTBsgX56@CAfqkMxWOdy zn)i7~cPhSey9Ds9u!Km+{tX|o*=#6}n*3#wtFUpyH#~*Tqy-rqBh1+gUmD8v=2M2vo#kL;v z)CO*9HC^epdVuF1DF0e(h)HsUJL7%4IeHh!>(Xl`KYEg_b5RN}7{V>?oirulK-+b0 zPR`r(N7Q*A4XO&{ON<>um4lXkes4MZA?T5fOXf#b_XTFhf}qzw>7fp&wJ z4~($!?-AP(myDcU?zI>V+5+2m9aeC|SC1{jtK)G?buBIB+nZ}_vD+jg3kBe<1?>h{ zX8mOkf3mNSmCz$kwhJ1x^-5_RV2sl>ghZSzOx2u4k&NiG`&N~f8z7H?()sV0ft~Z{bX1me z8-4y#LVebSZl7P)%mIm`dpp7zbHRY>TP*4L2B8kD{~NbA#XVcXvw+&r1U-WsYosOa9W3n>t$N*ZO$DSK@ST*ixN;z)HooEY1J-YM z1Ka)#IHV9rp+VV4H-xmAs6Iv6=9>#gnf+F5l=x!-J0@^E*78G7)o{U(_!U5uv>7Xb zmwB`~O2lJW4D+1#oPJrmo-O=K_B@s4ia?(?MKON16vZYW$>_0DHC|{5GG4d6IU3g8 zq{hiCgpI_2p(-RV0(_XW*>*OQOPnFVkd^w#{OuJ+!-_aQoyuGu517nP1yGsQOa%eqfR$0g#s(Kh2g zti9O+=qc@m$Ja%5k^~jw`1QvXNy9A2^NVRRo17-UUR5!NL>m}utEuj7x9qb@dM?ZEI8TomHB|;p%vC2DFXoT=^zm=8OyFKzcHzj`?)d_F zFVxZqljV;`es#mbHX{tO>_|Ie*aAF_>I;kpy$is?fUlxAc6>nnt8@c^26Fv?))$b~ zjZ)@H!wyvBz4xuE;(mBAhp7DZBM{JWJd}LMLcmh0Huye!t)sZSv=gj@!m-&J!#Q%J!?$9a5Dy(vA=_;-ImULR?)2Z-*^;t5A z2j&8?IsEf00$1O8Bm@^?(K8%SNYFsiL@zz_>SpC|Bz9EtXD^@Mg|07_&!79^1mi`| z)VTb4zCk+!Hjc?8aKJ>ttpy@YC^mJ8q=`IjdNZGuq*wBd#Hv>A8-R8-KkpDkRXSa( zISuVs#|CG0-~OdbUo|ba3J#%CSO7sz`t~TSh+Ca`pZ&CB~ zA;+40f4Iz_)XT@v>_} zRVUT8TmNolW_xvWX;!}o!i!nWa#eC|@Z7mV;uVklgrKA<9YsIt_>$g9OfNvm_YF}? zSYp&e4fp)P6&&}8X_u`+9F{RT#cI>V7LnM9Ze^5;N5eF?^lrhYZu{@;$%Vg%L$?rp zg=#Dn&-U*o>KMqNtY+&yKulNKpZOF~*P(Jl3Hk15h(Hr5XZAVAe`+d|sli-9C7~G*`hwmNPgyw=HpwFkOWRl? z{1y~Pq7cxecYb6*ZU&#ofRO`+Mp35g)HzLbl~2B%+@aHgF-oOI;+oDx7Ibw^4OEn3 z4>I-N{AIdc+h9axW7R20yy?RIgbTsb8_--d0coBeu8blt^I`(GbU~v4Iu8;4`P)7n zHXgAA=Vsi(5~YOouL8B*4K_P~vPmGlJB`vWc)VY#N(+^Ta5|Y0bV5+wzcjk`EN6Dp=k|2bsFi}y8dg?iVK&~*rr7Cd&^<9-{4z8vE)bT)SnRH;tm9v!LDC%%|KwDc=U0#Pf`Zs}ySAG0H56jpzj< zUw%Yp2a$t}@qm0b5Gf8ZDs-OvNW{K7T<9N0g0Adq&b?ohv)n42CjXYJ4R7>!j!yB! z#*OiVkS}UVr*45hKCMTilU7p+P!MT#Q*HJO7QCUFN-%D5VNeoObZZNrjM+%5geXFj zkRIkz7|QxhC_rP0|8o8kbfA%!Z*F zDx5FXYyD6wa(#NO8mNoQ%Pw;J9{o1qMg4X>xXfyJqM>T0MqA{A36u@a{5-uYpt!ki z=E|Hk%5S{fXkp+SNez~)yEuXST#^aYW?!r9>b^c5GYsGZQF7oG;}lBD%g3Z2EpNUl zaptzhhU~gnqn-5d!TI~OpEsdjFFgtm55y`9R$!_AL{t&^{nP2#rAD*}wyH{2@U|5+ zl2UI$5q|exYilnO-h0XvALd97JzmJT1s+;QrPGM;@B-n#WrD+jr9-3K z%)y#lD(VR;LH+MVYVrW0VIVFh2m-whC4n$S9SxPGrZmmTI@Kzi<^rO<-TbA>#pS2M zXl{td1-r0qIVO=7U0S37YZG0l9CvtT+G^E{l-_6A^TQPY>@E6kSRqc9Q)Q zu=(2gl51wD^ZI%INUzMynV!_U;n>uRW!7pyab<1SrS1%ZMRPf40i+se=O+k_H6lNO zVsk3>gLcXDPRr$Fp}v9whQ) z>*Xo$_hYz=oArL`alhY-ZR@${X8v+AE#H>;6ge%YX_Rk!X-YFH{^cnpUUjr`g6c{sY0ynLU;3p?>~#5{(q-JKx|G)^jI4jC=Qe24soO9!w~KOVa8TjC&isx# zZwAo4JIGdF6zIMkj{-ikExk}8@~rk3ugaT7$p=&{gGap&d>0Hq4bbZ6V z6!Bch`>kgcW+~9a`Pnaf2q8dU@{s`6Aomx_DCw$<2~;Z>mYNkb@*@GwEFt6t9MHNP zz7F;&C`1(HHJ5>P1OZVx)g2{~eA^`JdxB=r$03nr#d`O|O;-Ebo++szEVz*~4~JIL z^~=|o$5fJUe;xCc2w;omOH_S0)RI(`_PHYpiAaYY#~FTqfly_}=(^Sh(j0 zTsYAg^U$XVx~Fd%wmfN9-~7b19)ZtZ(M@)|cGtGfF-JlH-08K^~YkEkYuf&@?! zQS-O>t-AX&H5~t-f6XE%XMq_lnVUkEAlc`K3m(S<4wd7pMtwldWO%z4b<<0jn{+zo ztE+842L$b#ro96_Tle2-y;Aoc{s9yn*etXLBO?tqanZn3KrykM_<+f1wF=q%%`aTO zvT&~F>)AjuKSvt_I#|=~JDB()-TbpPj{PF%dTP{VA(Kh()g=*(&%Dpg)pZ|$4stY; z`MTZq4bAdz)(gl)GNe2d=bQcirU|tM&kX@lcYzGKED8`SGpwf$Qt`$E`ct8XQgIkC z5d6&1l~I$gHWuxz4sqc}<-1L%7Ay7@0tF9tyq5@)OfYOZTsHr+)|1t$5iT<>=XZ@WcQ8T6&V9vtV|KZHJlKr3voDD z>SVDxlg%^xCy_e1x!hY`O?z#a`H2Z8_>B*n4Ud#5eLlVhE&p<}4Am6I-d0djbB|{^YwHYOxp=}0#qp@&DKL_4K8Cm#nwB{Rq+5#~A zTqAC`O<>a>5jAESrD%C(la+>Un&6_JV_*o4k$HCFznQG{i0i>r{EK7J!u!E%ct2Qv zHyKc&6mLY=rpTHSe0S^C8t<)ZGst+uv~wzILmYmmOQuM9Kr($+Q{D8xf0N6>ttt># zjA7w+dKF#V&A2D85SKf=a-5F{esKdRw!0{<0k|tKA^9^tHr(;#&(e+HZvJ)s zh}S|vz^<+|P&i3ncDpKv^Ltf+IAxQkG`p%5>UOr<*4Gj&!!?FP4l{Bxd z4;PL{wE5Hv3&M=cV=PO9LSENsB`G#rEnbX-UB<9>_nmO7Ud~>=#1k@)yAuvOdu*vE zNf&Z~k%$>CdTQ$A7B7wXYaFWl36NvgXJLlS;g^{p)=uFoW3}mq+_ekj=0GUB2<@}1 z@YXGRPLB|PmjcR+s2e~1i*qmwrCYSRK8CK9340XN2d|ZSGGhw*_}rVjmZg}Yt$dv8 zR9j<2Y%)?FoaXZns15+)n~N%;oIK!hL?0ePRcv6d&+SxK@G)!t&h91?uFi5DPk(Gr zM`NEPWF*~wiMc@tl%#3uTgcy#Lzb;$vOJ{j=B2w z(yv-XK-z^<BB(;pj9w}-5O%d?ElXf9 zMf-YsLWRZgCnul$>{VA$H30!Ov$D&ApQLX5{PkNgQ}{H!h|I}8UpLz);~hGIh8l%C zWCrk{v4%lp(Xp{`ST?>2^dRQ;VIzScX3}7x?=Da(h8hFGf19mH4u*Djt1CTM{S*TC z6q7Av__HF}hBUFB9Wme>sJ7sSYe2yN(;U1N0J+z`v_)qot!lgYM;~|O2Ky0<0BuEyDYyD{ zjoiwn-paEV%kGEd?v93ZqdZAstymk1cLwVRWxhL6VIkL=qQ13E0V*Gb94LS9s6lNL zO6iI#3!r$q1Mj<+KY#lcu<2k{BjvFqfUM#Q{e4)~TT_X3EguVLusL}Vr3 z+*AgVqq^uMj|B-8MPyR2=3*d%V7KO)!JB8+NA?K3s=|+=6IdHM|GI`^Q(d}Uds$x- zp=0%df6dVNPVPt@mLU^BkX*$VGX-p=AKn*c2OR_IHd=YrB|X}*f^nuKwIn|rbGptf zy^KSPElpftRJf!-jnbZ?8V^3GxyPn!S&yIzfd+>n18D5p_K$`+?=38KSG*|~^t+z< z31zOoYm>_g6PJx4aiX3dT$gz8hCd_hCz{5f_#OKPt%T!)F6ORGSzm$Yy)+ca@%h}; z#f^&+34`|vR5Okq7bAjS(q&4;@Tq4gw;ez0TLPviqJ#P(e*UzhjMs$z(zk^bdovL5 z2?JFyI0#($u1eoL%hmjkg};i}+B8HP`oIHP`mN8El*5#K^z`n?d{y5WG6pnIdj|!( z4_iQ2p1_fYGWal}r{W3$|GpoB3L?4PY5F>C{q3OVx&W-2!iEx3?8E}ZH(F$fdP+ba zQ4R!7w(SE%|9`$Q8p-!fNIx<4V{Gim`=46cs;X0ufpvdA5DEvdjU0^r0XO9zt>m3$ zfX3tlG-?P;P78KqWY?|+%Edxju^<%6#{(;X-pu9MI32@m4LL10XF{gyBX8)A#(mq7tvw_#}{ff}5ES7;9?!lI}I!%|r`1I?e?ep&drTO|RYg>k!2 zBDZ@huVOkWd4k`$%OWQtA-Y5heTLF>+UOY^fIM}IG38@ACafsG{Dd5fMjr@4)zeB$!8e9|t z)^hK6XPN9opiPP}3&)Bh(d+xEAVCZ3jS4ix+547nd*&AapE+D7z@FX|i@_smCo^XG_E`cL82vh@DideJ4arM=T87WK7gT*Ch$~SA6!wQe3upZ_u3iuKO5#W*Wpv(rL` zCTVsnZYf}%l;NZFDBJ%<+gpZJ*}VP2l9CeANcU1YM7kC&uylbmNOyOKG?F4Axu^vK zf`D`j2#ACrN`unfA+cxO_wzja`0sbW@3;5M`rskgH8a=DIWy-k&LiuPTb^VsG(s`s z4?k*R%}BH|`Rvdkw@}N^^PkAS*SHUx;AyF(+2yO{bs@Ei-`?h^s)0S0oU!Org-L8v za6BsH4r<%SU(#V001g~fl>{vfw+9@eH7CCJpZ#6{ge5OUJk__5I1R}UT~I=Uzf+lY zfF&0ifF&^gy?TEyMz?8Q^XImg&{hyFWWr9*f((oV9~|fpppxyuf)twmluf@fk;bL` zRgF%fU?sPEvvQ;SYzonph<3Uv1{M|<=N*xg6-1k#Hk|G7dV)18mOeD;S^ z(OkJ98dDMx$e$^ZCM=`(u;8JP^^&%bau=8;{jZpd`-&_A0(g%J{1q83y9GIoV$s_< zt6fe%=$@akzEQ16xQ9U>F-*Ut=Kg&PG&xBZ#G;A#iTfGir@j`JCCgdw7?L-n?&nH_ zsI;#Xx)5|Dv(r@W0W%QV@p><9l<`9mEYv!5$QeRlvTxSJSjO&9)}zURFYD8WQ`qHr zF_^38g)&YWB0-ni&<9T%X(=1pXzH_0PD^Q@PlH&Nq%E_eIMFKcDC<%rVb>_}a###u zEp30@XVq=a2pS932%UJ7dcIU>-Pti?7$iNYV{01YPYk980eblnbgpfep(67rCu@_| z=X;NxcPlTOxcGgJLm)Hc==uFpv%@l$4SpG4NUBrpKj|WJ|9RS9($#v-AsUi<8o6MJV` zV>ESR<35lUq5AoxoOdCJYQwgy(PLi?;vxA4h!n#v(aUQQR>gTb^KW{VIvrUS6fEFN zO7#!KTZuY?ZbdZM7@##~y)Kj5FA8kGm!McAxGZd7OqU8|^z~^#x+w;5CYXeU$zxXD zFMqV(latFlq{lMVcrVQ!6go&*4^1UjLIab+M*|CP4K=zRq1x7yzMFnZ7t3DRLQyU< zmDd-rkNHe9PXQqPV;#Z4Xm4;wN_>$jQm2LFk|?mVkn>97&|R;@%&4wyvnweEh=XMP&?c#2 z>FU9NUQyLeKJ{9r(r*}F+i#|iJmc_pi?S-y95Ff)7qd z+v$9OlaE%@M`5DAW54^)WXU+?@V0-|-Yry0RO~JHblIs)^sfu(U)8poYc!lxw3$4c zPns=M|Z^QOL7 z^8rUsQFLE#$x}3h`RoVnmDSb7LfBd7h6iHGaj%=6i`_w@JFiL8E)9IrnGjUU{noN6>9{Zu0$}yz`&-#Jz*i+KKkEuJF5sAPd1B zAML>fPKmo@8G_dLf^IIXJzAF8h3tOy7AkM)eC_$FVv=u_nv0ye$;yudnOfOHm%Eb( z7zX=$MK55bvJa2jPMtpB2@Y}AzP`sG^#Blwj`U3+D5e!ZUIpORGX$M8O?$V{REnGI z);xFg%l#E1PUQD3Bku1gSkHHBj&%sVwH)<@a1GLLBaJbfl-k(mC5xc7!Mz>9n%$CT zv*DK~;nJ1)-iF1wAZyQ+AF4WHOD-LJLwaxfS1e#agsz*p%BF$6BWxD*wYQ@!4p7vW zcl)Jf;2sq(BCz1bY$C z1Xl~!A2y%8kSSJUFJr}{LQ&GD>pi*I6MQiU#ffC=vD2xX)Lzn|0dzT4=Fr#6?{ zyKIs0+NJ{Z?ohmXeT47G%S&2pt<<&fr+?4yO;pT~y_k2&tIEZMH(R8rB0`o>&HrAS z@H_6YB0{xI!)A6xvB!}lanlgX!8hPtmTosSe1nT)e<~v?aB%ryzQDoDH&xJ3IkJ5$ z`0V-L`J)Dw8atM{dxyB>%dpISswhEZ3eisI6>i~Z`EwJG^(yD1DiB_yw7^l-gL4rd z`ExF+a92@%K*6JVm5Z*8_63+t*>5i-BLtE6c6CE71*H^9bHhgw^3F!U%fVxSHBAAn zhkTzLZqjjtGcG6qkM`pO9~iT~hZ+uuw>N9#??_xhQvUA$m19zvw>RQ#z&=CGe!JpDcE242e}M1mu*vsZ(sMtoeXs z(R`c?#zhbB9IyOfp*2===5hYn4g+&HNo$WS>LqAP+b~?@x8VGn>ob#p!vT(S;A)Zt z5_Tp9X1%eEr_=VXs0>0a6%&K>j|Cy7a*DuTH#KeBU&N<}$S!E=iEfq)u)S>c| z<=VrKKM#|?KL$Hqw84b0&Qf z#!+!|vBm&0d3~S9_}lF@^Zfu_G&wUfMtrwX{;4~5ZMK9jDR8yZN|y+_^5Nq(`cuW0 z|8X}77peY$76f2dkj8uk^nzGG1r3-Q45O6>j~mLr*KPvSv;=y|SRg}P0`f=^SC9BT zpxKV^NV%cM0$D0jt&J(jPhl|Z5T1N77N=5;oX+%4lv7m0yx)7RUVsyLqP#?M9*nSh ziM-p%zC_WVfi&Yah(HA(Q(!ZJOdgzBaS@d`9jfR1RM60OBYOR#SS{UMLaY*G(c?%+ zNJ4>EnCUPls8LxOV2~{V4>HW1GI&|aBfq|TMpr49^2@p&gx(%?|1*$&; zY%Dmn{fDMi^>3_I$u*;jFWLgetieN+H8d1c*`H2aYh;VUD!G8^)(Z3+1`o0_WbkwP z%Mft0KqsrNUIlv&uLw69tI6g3vvj`okgX>H7cFmr(P4ex35v~=-eg|}7{0_$7{XQ=8a>1YSrK0*Q_p==sb=iq& zK8)8#O=(x)tFxtFNF!hojouGrl@s+if>UxC0>2;a4c7w;DP^o|hxyivR)21AnAnU{ zd8|htbpEpVMb}3_kqbbcmlM@$2ogu`WC%X}UJuN9E*OOlSS;_!P2$Ak6q=9^1 zR|cy-dW7=fl$B6oAHx7HVfzFwRd>GU){jA!3jk;A1DZMK2Qxrp+n729o0(0<4MCr&?38iwvZrB_r3}%Tur2FOR?|^#bx-+FFnQcf3M8+`u2D=@Z}> zQ+&5=2&COreJzi>sp??Gs^lkU0EU~$VjMz|>uykEHmP4AeQA?#cKeE~)Xw)U2a8A( z_KP}y`D?w8=q(s;fcH?pw?2zc5ZOD)FX zBu>D5>^akZfb4=x58+zfOVhg;5;&q^NRVpZT*JI8$#x3O@9pagd+zAfLT9+QbhdGF z4cz?JD(VmXfz{gGHgXdg=-%t+*^SKNNFSqKRyQ ztWitgYW#0LrQC@5;2W;?n0=ES;Acb!PIxa&eE|{t+g5nTukW-2U%qJZv8talm-auW zVgycsWvvM)Aq-$~pFPR<+iN>JEFG=GAIF~V=IR5V=~_Y@hu2TBJ!BJn^FCl?6l5i0r-Z)-|_QpAK zY)3wy@aHe{0=-tL9t;;E8aibNr@Dp96Qyr-i{q9qb-KDc@#af;^Zfv9LUx5m;Y4cH ziM6EhrRCxA6(D05mpph;p<8fVKsX|C6rbh9P4G`=VmtWuDWmYqZ1Q8{c;iwI<{K0r zNH*Thq_*WWD!8Y`xZbaBq(MI)BVm1S^GBuqw58~9c2-uo6eTbNdAuX$CoVz<_ao$Q zG3Gpk4NNcqUH8i_Nmw?;QK=2Bm9X=G5d(0jqLPpltx~hKm#Lb6*5w843X7)+hHORr z)T;9Jf%N*xch{@TPa3P6HI1hOo%q<=He0UKBeRa@liTi(xd*%q)e1*(?K6Czvm`g0{99J zzC*l#I)AXL9QLr_(R|rOTgu~nLwv~xinyBx$g4>u8Bd)9 z{0lAp1Ekz>JDFC5vX4w<4xeu5%Z2QZkjB$Oy}pQGC;gQxQTY!Y=gasfEE*#u&j88F zrG+;-+3C3RT)^a$LuRbPca5IKphr~;WV6IyIiExZAL+j$d$~dgpS|ma!tJ)N|3R)SWNi){6B`$is%QzTlHE{wsu@EtrKVOnJgBm$rKTmvArNnvhJ~2d9hCKFh zOa&|x*<1n3AxXcpSazAkCrYI#Pc{D0r9izYN6SOYYG2oY4;P9RHT+6+R z&}m#8;^NF5{3e4hmmI{Je zjR3N!vQ4mJ)uCT-S0S3;A!pQWZhxrSV|Qq|{vWSV#kfoNwfbt(omLdv!x)171V{$u zb5u2)W3R_31Z8^d?YC4X;zQ{$DZzk}Wbv(8S^Fg#_M7>%=}-*@?Jvrb|B)q1_5X%U zm%hY7g6vzn02W-}gfg~&Yu28WN@8;zgiLK-;F0sX-Y^)!p|yX`T1jv6g&@uf*{p+e zH!@w&aP}lyd}j?4zqEVZ)iz?wB>vccDT4CbjKnq}d^`M|i|=$ewoy>x=t>sV<2k_P z^V4ok2eK)cuZ})bT@w%VLlQj$%cG4GTWc%(8QfFW=o4TfXEnHbRdfz}FTOO2-eg+UAaM9B~U85;1g2D9$)E>J-qG7)3^>p_iL zmuTc#5dk=#@BC|6je3mC|6h+8MWB}#AC;QVGni}p9Rf3-NW;gVwLqnm(Hv1`DR%PU z6$PE2f1Bfg(uy{759EtpaHNWFkkU=|NrO|b#MkU&ER>`TkXySIwupaGBvXn$kjM;C$Zt2H#;(s{go4T5qkoNcs1@hi zPOfcg#f0Z=3~HF3rm_NuhnK+W#{DMyknqQ&oJ!Ia{;4D~&^EpoXjXZ^N01<^ekVrl z((uEU*)nV+WgrP?|K%+_7Gi+!OyD6^d%)I9(FY_Dm@!aw$LOd|0nhA*of1YMoe}vx zGs-}zJ#?8(b%Rs^b-9&?;35f<$8@!78iH@LqM^6yIw}X?D!L@%Ul~)P2L4u;vy=7B zUhC04aH)?!ovTvZ`j)=c?u?e^REA!~nUAL{H$r#51+$g3dmaM3v#Eq^lGHJ>M0LhJ z&$&8Xj)7AvJ@7cN8E>ZL*%ANDK>~Ea1LRtk|DGsCwX|x^(&DUMp4RP6MgWRVJj#aY z_%~ouvFr1IQKBqC2aeu-pB!gol(`K)5daKk-S6UQW237;HT2KH0jIcJfI zA+ax>xNbA9ULW9pax}~h9&2nAWo2XYeJp2(vT=0UJ%BF2dTYarPW|K!xgF2Lppqz zDw6agY;04WkD{ssHh5`LIq^u)O*tqL3Wg$^GBdKaD*F9>YF8d4c&XHvqT{=(qZYK6 zMxODg1N^&-_61s6j6Og|Q2+IV$rGYr^(#49)wuJm$yu^8-dd`Va z4(1T?*mFvdyXvjiotjpQ%Qs`HD4#$^y>|}!Y4$eui6Zc$5KvXw11bGh`#;{moP1%N zyezU*ldhlAIbNdw-MGXG7Uq!f&P1HO-3y zhG{a>jnA=-wx5yAp3-I&QYc1+hXCSj$RG}RZ=nDTx^etW_P@CR_TGUuw;EUH1MX|; ze{Cu3jVDifa%G?OZvJQ|srhCN{#HtD)f)rV2VZAS`ZVv?v3`WP4kPz>@x>QD=(XfF z7B#e2)Ig0AT|Sz~khhLeXk{Yn!IyXC9u-bhzud0>o1E)4T`5e{Y8ICUSAzW7@xs^T zW~chUepMQ$g*wvIV4qSg17Tew;qzdmE9^J3tBG0BfAm7~nCYJh;~}||o>=X>V`dpc zApskG>;5z+CjE_A7%wbtL+9ZEG6+b$;UA@9lZciAZIt0QgV6e#y z3MUXomGyxMo_3b~4fR>U^Zw7rLg>?c;9dR|-Dj%k<@?nBz#_6wUjlN!o&xN$uH4h$ zJ0UR4!}P#hRv=GQl_+17)LHUpL(@KwZwrC=m~+#Eg%PI&#)4u849Qq1f7c^`HX&k_ z*yy$Q>EGkus+OiCE)O-(A8sf*(_XNver7wLP=pw~034G2KE?n2s?n35yvBVubMNqn zp~X45#`QWF1dRu>{~t4<%2T{oC7tx$1r^>O`GMp$3kauQ!kf8kk`T`3Yjq3{x2cwRO$`{3<<;JO}05dyf zcGomR{>N1NXTjKeS6W_7=Kry)JZ-y?S3*qRHh9yG!nBRNOYiH4gn+slXY*JJSwiFr zpTiIfr6G6-q;6O2dQ^@H6l*&lJ&Y4aHH{euecb;;!Ts@}j?-RBj?_$FXLjfLH`%p1 zZ!?-a7t$Nn-uP?0#=dDI(aV{^!GSdqX7zcz9++(T%7ac&hKLT=by%I(qrI9ifk!Ul36+DFbZ@xvf$=rl z;YLi?_zHVD8HX1}c`*sH%*BZ(70f|9MI`3t+Q;WxaNrgCluu{8lE1#js2oy1^vrVE#b`tvZebOq5Dk%DZ<=}$Jt)~aYv;%gyT154 zqI)uFLPvwcSxokcoOEoAe^2o4}7>IUtOxCA8+NmqWPEXE(e_GJdm2#}MsPn~?bRL{4U=(UskR=WSI-e6Pv8LTBIIVwl!}}{Q3|LXTJY92M{L*El8h5!DeEa@uAahsdHFHHeIiA+iPIzJ2s1a@$FkHY)7?$NX3v||!I$lZ82sT&P3*>Bp-WU*X^@=k<~tFm#3oOvh579H^8=IwEk7>w))(6~&5lM9EHOU#g%hVjnlnxg zQ4#~<(rw&b)O%GF@P4VuCpnmLaM0&v7lDxzszQeA(~H4I^Pl*U93%(<0_}EMJlG1Y z+)+&ISrNsz8_Y`mdK5nNyO(@$6!cgIbNvIjW_FBj{a8Xy=BF`fy+RUnS*WrwL7tm> zNJN*?OnN^z?m6gbIrx*RiEvvPG&U?C;0AA02sx`jp>+Gt_%I&EM-!)!y!mR&qF{M1 z;hxgLD`bYP+4#dAtvVYL^$fnQxc8|ASZJ~`@*$z*LbWC;s(tb!LD8e5d`2#>kRD6| zm7&CGwpy!nU+yEsJg*o_G=oThRJtz&4WqSDj9QM9s zA1%$2huf#$MhmM2bw-o0Oh8RYP=A3Px---89=~&^c{WUk?(ez=M4=cz*_QAXQayZ} z6`?OeY5(?HmpqZ=D*#~lx_S!KF*4%(WUh9c5eeAux@+SfC|$SfQOE;AYcp}?4-gN9 zI6=U{DFy@XLx;vx9?TR3oZ+iMO#)ONoAX&w^i<8%JDF%zqJn^cU6qLi{W0WWXrl)? zri2ps*XY5c(*B}cP?$kGDDQC6AYbpSfc^ER9~Bcs&GfAbCfajiVgxBdU`2Ltgi#KD zA**?bhknc4IaC95*)Pc5)!sge88245izOikmxD_MCq;n5En&b4=;dsqD{y{?tPa0h za~lUvhl?>BxmwiAfD-92jE-SLVKsaUGV|G0qRh@nKMb5#s6{0u z>413iS>AA_pi-a)0@y+rXc)Xko-gp5_x+&>SMXly#<~JR3?Be8$)?h;Sp>2hP)h|g z(mV$Emu=^j8?4pmAe;T``u=)hq8TX25?k?>*CYWHemDxLwqa$0FhSzzh1b60{LkB! z;M3}?+4PcrNC9$SuEYZBZq!OB2s`}bi)WCY-&i*GT_K11fJoj8e4}7UEvCu0m_8t1 z9cm?4Lb*9!9M{8-)H4F26jArpc=?ZlMC5y~Sryn6R@zF37Qe{Bu|o%~1qGvH^}9j6 zk})KV<)qsXC|{bt%MylJv9WKj#C=b;u(T|r6q(5r_trSEE_*7^!gIx&wsCW{quij_lY z>HuF#W}52!QzfK_oq_|yg!ijC`>wmPl0t+?~C_3gkL>|8SfYr zXvW|0+hLx{kuK_b&-+?`Q$k(fjr}&*S^)xb4hm;f`yN@HP^-l91{zy44kRl)Pyxkp zCe*Hhochk~j;eCVgGB$2X@F#0k8Q{OG)riJ{Cx2q%yr>&JHU%{L6Uz)p&smQb@o#l zCxu)eoaUOVujvLfd7lr0l)mf6I2#o*~Z%KNw}zg>7P zeUZfny~=V$r@-`gl^nCAp3Bz7yyX3cQ|UxY_eSNrnEJ2R9( zALCn9&u{md?fE%+5O6)v#w9mEEO+R|X8%!S(&Ha=0{O|8oLfT^zL^6)ex%Sg#2-&7&sm-uFtpTs zlAfUg|Av@bmoCtO-lu$OM0OddY<65GDge7)lfLaj8OMHg0KYCg2xbz1nlJd?G5d8_ zaTHdVPZag?S5}e)((;i}GrZvezDigL)b0_mtk_8j1KP`egqTeZUx-%(v27 z@V4&tgy6SJ)Ze-DSEQ`0Ow?mN>F~u^ze3&*z#^I$%3ggGYjyeCata`RgG@5hhO;j~ zWYMO*{7PkM&Cs|=umnQ+Z%~uu-qYx9`8;A_S!;#f1$QaD`ZnMZS)Sy!o7}O+W;o;G zI^(%MSBQ9$C?);PIrwEdEF>P=;tsy>j4355GUZACmIK8K(Bw19YW7!@f|DMOO;6u_ zWW)O+E8iyeBjqJxeSK}{5*%=bFMI$jat>nL8?JZ~U!g=>xdnWaZBT8g7t}>@BO$+5 z0EzrtP$Gul3&VeQyAcFx zx9gPlx{TqRATwlpv@#UGFujApILijH(_m$9*S+X2Q5i9ALFE%;Nj-{SP0Wzp@M)ZI zGGryJ#ncp4t1SPAlY)=R+afvcc>Sa{n{>OmF=QP-)r0P1MY13FnpfT$xDkG!R4-@< z%F@}$`>i+Dv8H+s1&}noRw!rutJc*l`{T&7_y0BfWd50b9$y7jAca4-<%=C($Pk7v z%@GW*O0t)1jD0pjAof&-c>6g;|N2Wzh$8C&{RFu#qj&@pVo0Nh@$+v3Si16SFe^k6 zUEpI9roBSlP~(Ki`qoEEZjZV92s0f#V;XjnLp8K;)jCt0Z>c5-gi7-2(=1%uXKV=x zuU=jW$qG2+jitsT2!u$$8Rqap@FBRulGwl;S`9ugVnuo%87HtGv++O3N>{$9^apaM zrZ6-xU@7#0W&}cuacR6xPtRL`O7)qoAlYTQa{exH+FO$F5^gJ6jsuu^^=ovG2p*O# zDmGDo&asj@=dsbai#IE6>1UKXCD0;G#if!kDksd9D=woJ#Wn8`(v2BYM;m|-(U{JP7|>#)-602 zT#{rqa7{=4Oy>%s!uqoB!2XoYb*$sFI&lD+u+yxrE#0Tk`-;Mlgb#DympkYzxu)dP z^{Hz$lhlz1u7Ru$*z#Jme`q#{E`jXJEz*rc5DPb{NT~of;WKoA7v@S}*AF~({53=Z z-dYw|K`VS8zuqWUwqfPUlCLMA^vgJ(S^M{x#}1BplIZP8)GBAdIpuwNX60zGf*?)d zDAm3xZ!R%;JtSq|c;kCo2?AG``m$L0t3zHm(W_Ocv4 zyd;tkR>}6oGEaf^@Gj1*MtS(eDHYlPFIqgpVjrXM0&L7xQIz4CmEju*DwyrVD|ER9 zTH#5Vr+J^RQ5$&E-^Y%A{!EbXOu`b3-|zExJv1q~68V;%S({|j6E0@^#rc+1(auv{ z#Qq%?v$VNjSaOl|@$MUOcU$j~ZNa=2!N@en%)wX-=zY%XmscNOP7v6wh|T4wp;p$M zb>HzRxCO}T=VMQ(+v`uWdW(!m30;pR$`kg3N=MWc>uSMS(a!&zk`ISdtp(o`r;Q{s zE>d)Ur4XV%OrXSIqbMdPV*DmbgjW7#!6WGzC&z~<~WuVV;&eynydPl-=B?w}Y5Ru?1#Z@)gG`LB1Y6U7&qSKkSzk1tYl`=pXWF8q+Y@l_YY@f3eZnE4j4yb#?- zcj@}~V}guDKoI7~JF=Z0Kn4S~P%t1M_^~8m;h+b-2FVK0BL*!Hxfwzq$q?_`WBz-% z4Z#CPCmaj3un-fFJVAxayKvBWB}RN=%qyCI8kHxV4=x4L=z1ZHi&G%agF0-Xks}VT za54zzV7_=tEQbliJ=I*>J?(*)$FK-|s3A)y>DkwdQe0jPLC4gGOLG;VW~H)lOkCV^ z;f7YO*6gn>mS4Blke46Fnm9CbD|QoH3Ao?2jkV9)aiH2zX+5;xC}lSoV47g@sXS->TV2-&_O>>>+P9qPCNe&^ zL(emTp=)0v!v`L%Bj$Q(aB~GikSEUs8A+63DF>&svh9VcSGJLLgH;p68MfK!dao&{ zj|0*|7oM{~%)Zd*sw0Q=pI2mTDQA$q#eJz)rvUm})<28=0;l4ntm#&inmTzrE_ko~qC*yo6;$A_1QZ z33i=ao&aCL2IDviZg>pI+h^8I_t--m=bDAH9y{-P(Ax=3VE@WxlVA7W0=$_j)!t7z zdLL|iW66IuxpQRj#6*xZQL@(yLp~`$JMO<{X@QI*EsduIqLbtj!Nmq-_CLg&4PI_RR} zi`<`Rzn5ef@YT#RzkqH4mAPy(GqjaI!oWPz;wa|P9cG1<3nqV6ctQZ2m5YL}$m>Uw zyY)%fikfk1bv2&^xc~V8?Xw$q#!1C`B9re={HNyfkpwWyaTWC})P9?$Zp{zH=Bm!B zs!Z|$(B|7UqU2tDy@_OA0|SuEApqW+2t2ZBy!VF*G{TUPb5`@4u3C>4k{YldK;i^b zqUkbk0mt#puSAnO(d5@52||z*Vb>Y3UstgaQgmaSTU_m znk#6XfR|Ep0z1I}0+<&E;OO_>uH(}=?+DnPk)|CDb zA`rJLz>X=l!l`+KJ?&{WZzrh5SRQd0F;%QadE(supbTw)MzWqI=zU^h;@y#Bl;2Q{ z@rB!;2I+Qn$C%hG5@AlAQ%f$N)0yg*7*gkp7%O*~xt_U?S#*VA4Q4BLoBLn>5IqHS zq|(6(_Z(LCr}C_>C|?1)*N>(&&{y$&0WT1BZ&XneAB8=S%@#(6maozJXd*h zA~GX7{Vx-bz6{}+of`b=77*FX1lWZ1QmT+`t)*GK7K}B61t^iyCHBb15qbJm7>TC-k0FIL4 zfJTxjNRA<;{Py!Xu!tNOkl_V$d!Z0~{ht?sny4W2#l}ahqh>}CYm;OW2qxdy4(Zs$ z{(kIs>8*@Mc3He`UO>^8{-Mn#@cdUiwV<2SkLXQMiF4RIXr|4-!H@cgR;uIT`TF0@ zxt0dMrIb6{g*%^|7ZL!W({Roav8uQ*oi7`n{b?;@0`m#Q-wmkL^nLQF$?BK#qkohg zpBWGhq+4}>5_h3is&>w&3eW|bDHX(0a~W(pPNzPD75WiTP4t)4KR2|E&n0~Ar5b!9 z;IY#z@>}jmNSn>8;dRa3WYlW?j~*$rKd zM^Snd(8tzY#)O2&gnDmS(SnXjf2Z@RXAuaGMaQx#9c5IG-hNJ(TmiiN&5j8%6b}w+ z$NE^ZK;#^-<{;fZ%P|}eMAoirz$<|46jIMt0eZ_EXfAPoS(b3p zSoeV&C?PbMjjAk6p2VzBeCF}w*6=@;EkjzZmR-s{42Jd80%yfk;Ma=my!+OTLw2mF z2)Ca3GR&c#FQ|lsT;;12BxwYL<-#E*c2(R;Z4o~=9d=>?NJ!_v7X3J@Y{a(QAKc<+ z*&&w=%3%Zn`6>zBc3trmEKQ>o>(R(MUffU9Dc66%urWX@<8I`z4f$wNO0-GSSeRsO zr3p&%f2Hc@!CvwaJ_Q@0BFWGU$Q1nV@M4cZoPPeo-%fV%te7Fjiq)zspDR|MJdhQ2 zMwK84aUmylienCZrhdN|%HJ%P(M3s6=*8V??AS6|;$V2q6Kp3)V1dBs z&a|1u-yOMgCB)(ObiyTi2k0Ntylm|6)th{fgdI4b3c+T<3dfInk}aq|{y(SsltOVa z?($=AU>Q&JMMs&3HS>xzjly*7{R`fxEYffeD$!1V<7{Ba)-ys4Q+jR?)Jp<$RnJ

PHHuW4yRg4DBZ{>ymY=S zv{Kv2AbCSj=I!}-*o#mOo-54_e$3a-ByGjakd94u!=tVJE^0sgU4}><44%+Q-X(zU zL^=`-H+lUstk4rvx(GXeI?eDIH2TRtHCF!@DgT$R7{l?X8at~UpUw(j^9#JD!Re{F zK>Uy^t3RkS#m5fWcOUMTCwvL67ryB7KND#%1BF28ZC2%ws?XFg+c7WjA-9{ncJ#?N z`n7zm6`;@Q8*4siia=vH-as){V?R4Yn8SXH0dut~Q@q_c z_fewWJB0pB$x@(#rk&)3%*K0gguCMPzlg8-qR~=RW$vT)r~5-g(j;~I0>woXy+MJ< z^z3;f+k4TE?&LSdm(z}^TLP^QM~q-qYgby!ndS7%eJ`_Zj567#Mva| z;gUA12|Ffw|IGzp4L!uCyw-?kp7@-s*gLYmRXCEE91y?!P`}PbgJD@AjKg+cEW~R^ z=7=OE{=$Fe5mVN$#PFYr*Zc-jx1Sol?c9#tZM*C5(<^elYL7+ z?YWT%vhXxn;C#gkv%R6Ps$7(OXs7%2Yc;5&2aqPM%y%};%W7#a$1j*5qFxYM155bXyasmc1Zq=t)Yx~cCFT35vt+2*0KA2^_;u+VWgKADpLJ4^fB6D6b}GqC{QYY| z@cZt^x@GkX6?8-`?SA?t zU?^Eq6|IV|QmUgsm|%faf?s7@Km`y#~__=Np9E zMeDRF`T5)4CgkP3&Xe|&$%zV-QWvCTQm|yta!PNDAe10j%=?*^H48p$g~6R#mUtf%%{$s|>TSI)-x141$Eg?GHsdh| zg5!vGqYz3e_@y^FcJ1~c>u)!^&)b#VvX`a6zJe&{u>zI(`(VBrL%pY={Wr%vZ8nb$Tq zM{mLX311K6_u(^kUI87)3G$Qa1*?3en|BXpuJ4L)=Z0DG*fBgTDdj!%S7FeBAq8!{_O8OR9P9RbmTj(@^^U% zNB1uQcW(SzMIv!prcLt`ioIaI?Z(f-+G96wt!KA)^p5KO44t3U_G^{-1s-=8EL=2M zUJE?RJ!=}oGxc{Jul-e=4$wO)Xh^0~hK254*iG6H<^I5DC0M(kyU>;tz;8<$ji8`|m*l3$5AzA48uWo-!#`oXM_#&E2@E)IP3)4r7$b(Cs-H1xc$IKg0cmM3jC za_PG+Zi87q<>~mM-A4-T`@HY7;<}f35B^0n(S9(RTcs9gwnwcq_hxHc_M|&6kIq-F zd+p6cKRR~Shd6&Y{1f{6=l;-LNOV{i**qBI04>QLgk#C>}?H6IEg6#r+ot!eJ` zZ~ppF5LA0_-RpZ`Glqt*z0mmcRCp{|a1&p7i;XQK4kjP^*S@Z1lZe>QreD6`YIvvQ zzKta^@yCIx^%eBUOhxbwN1FFO?TYTy!Brteb z>4QO3p-#9CPpP=)Hn76HN`bKK4yy&u&)3HD;$Y&TgR0d!JfSW00l39OIv&m2d!-tk zHS#TE7vByz4JEw4DLL;73vZAH{SBytWDQ&;&r^W0V{tGI*jaYY5oyTx`%X)6{3ST` zxXvQ&j907~XP53oOisSX@Sd3S<=>wTN4(n4!hBa1FwtZ?#c^!TN{`*#b{Fb#rGt)m zAdegG5sLXtDOq%U#|XZS+mUDI>7hgMGgZbly%zl=8l-#jTfibixhf2Vn1Df&j%N zLl{e*p_fvmI;_}uQKgh~FXL@iJeRX%!q=~l41>>Tqoc1HD0FXhURE@CwMCGp=!H(> zksv;zTVl8yw)((|x$p=RWzPH;YKpe+rl;lPL|86J#DV6yj(j=bq&E1rVt1i*2v*V3 z=j68XFm8q)GG%+_i&RB1jHXY}{we7b1s8maZ&lQp-742{jU*m_DmPi09{T`D1?c)N9^^%`>=RK|9)EcPSrOmIb zI&=?Y+_vtd=Ooh2M(lyz1-WiE_}CIkCq9EkH{DET5PXXblo~Ri4#LS}yJz@Jkkk+% zb1I!!OSn;T#>aKhb!Z-Ix<&b0+C9G2U^Dto&xfl^hrRB0e?i$1C=i7b)6&KPG7;BP zwJTjK32Fhwe3nl9)<-E>Nl77K-9X&TJk9ayOXo479;W6!e3I_+Kv2pTYh%OKX6}q% zt+vSBNKY?Q_i~cTqJ)jhaWIwW`q9jy!{|61Q}wQ%G(DL&%qZ^ zTCbUR{rT_; z%CkIoE?>(=YhY75I~*T2nd=_x;B#u1lc9=u4KZIkoPiX=>r~;F>OT+{pm3}9C%pH- z_4rCqsloi3mw{1Ga#Bku1=rv$69{jM445gTyf1gBoPSRogmdauVe6mLghq3km&ggq z7Aclm7A&q98y}uT;J_}>J;UAGfkyc!Kz=trwUJL|c|86()M^z&sbMy!xfil3qsV$f zf6mCvT*$Z*SS2Bx1c>1Qez2OT`T!A zOKXuPwCC@qrT&e*@Gks%gERTpR&4>;ndJs&J(!=#c&j#L4+djCC`1pw_(GeVUUs90 zl`$4f=M*?U&MD&<(<)tKc|j@K>|Q^3*XL+)jPpf7c^93sAPB^x@*zuW4=Dqq9N+0M zKYD~d>`(Aqcr$>_mMaDfwrCNXNgr^j%s=$Md9gjtw5{oK5D=yxJlQt~dk-~v)$--Y zx>vE`(Jg6I!w0RHQoYG81~5bn8eN?<8Ah2zgx{+toIC~Xg!UZs)6cT9#cL>MUJPpq z+sgHHk5%`2+RWAP;!)O@$km*Av9sW#&qG5n>#YWeMBPlIvXF9>B7+R%MiXc4Cfq*E zQd0E1?dQyh!ph;Vy9B;WkUgo?l7w&ntGw@ir~3c@7t*n}h-|X=O15+Cb#ja(d%cjX ziiGSvbL_$)juDyBKz4|%q#+|p%T^@5_v6+3{rw9**Y&xskKdf<<2fJue!mS4$5GWo zmCRaO&7`}urQX=S^3>Jm4rg{QcY;Wcg#x7|d1^I=armV5JvZ$hyTx;xF=s+D)FN*w z(iID*EG^J}n~jEk5QPrq;VBWay3gjRu$tmjzdoD$>bvwx)a;YM{6Xbs@r${vUmF%a zi(3MKi%0!;;c|!HVjqu~3d8q3^`+Z-8oKF)s_pD&eqQOUBw^1e46>7V40HfsqlHJv zdagB-h%NHCGg4=_|Dl}vIhzY6>D=kQ8;Fs+YO#$KX$#+07L{&zoB=PWsUN(hNj&YA z7crOLF~=nSN^9Hy7-0;{w!=zfV~vz;`_14EWojOg4rLnWB*$PPOJeG@c&(w?Qn69n zl81wo6*O}_?b}NA;ZcvGmh^B*Oxh)%afz+DCybGF2qdY@`1y?N{K&~kmu>r4>|4Y2 zab`c`B^{xQ_s9)w&3ji{o_9=ju!JVZ$NPeLJ5YBc6c=I z!q0i3i@l19ecP5pYp+Ce7}5jRnHVGswZ?l%`7bF`+X;^78IFCGD54}j7D4n?i&^A6 z@mQ}Dx&~92_(s%5hNjnfF_e>Io}yFxVvgh2Rr^wKARRpMn(_)M3`dWMsyRVfg2yUe|+H z<`o;f#D?18{-y&_Oh+Oew&QN{v#S91qXj&L2pF7C2Gmk0ZTUv$T8 zWE2i|$W6HR*nyF6xvX|DT?8oOBGdl4Hw1dn`eJ{SaE=Gbm9XxpuqC{FEA>&&v(vft zlUq?NH@a-d9ZR+i5%EfccLx$$n1e7avg!WFq}BG{5wVFknxeSjWI2H0Fhdkr3m`hgKH*xJW7l2!Scc~k z0XR4h`3y}kA9Y&v#Eo73iSsp@Q!VSeCUzeNdJXNl{pX<~Lp@l^3d|`&PP7>Oco5Xo z)OzXBc;b!M$&!1&))#+)X-Pf6shFv)OWZ^;m;Iw2edQrQV4uZw&N!c|zp^N)r_^A4 zxdzf=tpM`7X=6A@6&-^g){rLZbGSDL-htH)lm;^f+vt1GseyiMLR4dyEd&=Zp&uCO zTMzx}=dh^-n*lv}uc<2Ly`QUT23V2f$9=UhntsWBiog!0L0P|Tx%8*cg11FwaN0AC zqWKMP)q}&h%2OXIo0v!=*^dcEy77NVPz~qa9Yk52sRfaM#Nvg8$RQ_QB=ft`hKcug zl>CUn1hF5SH#=WmGq9c76S7$MRBOnUImo$PbBhz}eUZ)#0oqiHT`N(ozMkqP-|f=^ zDroHfJWff{21v;it!&YWQUv<=V&$W@GoDri+#~>9)ARSPD3`wWpECQF?C4`XrAPkI9U6z)0OjOVwnUGAV^!BKXhH!Ify6hWOorrB$A*;QpL$+d}C=4 zYhIbUU%wT*`}n#VU|p~tL$)w?azmB;MedJl!p@$s)iHqdbPe+2G6Ug14LV^6$O49R#4bB~CnmoNzKX zqLq+cf19p<;%{+L0)v3za-QMecexDj@-Eg$`RHEU2A$ zUHP}}I>KJI9>^BrXyNm%NpDJf1i)659HjDzR0qE!v(niAfmOB%1LA zs)G4tMdX`F=&#IZEa-aO!g0T40f#=KLmGXIfMGT!9}BsA6_P{l^z7_d^9qv^-}x?3 zO=Y$vv=m(gL52p{l;B95oO@VU7~z3YyP*`SdY=wkXR5J$lC+k$Q!<&q%B!|b2U73` z2V&k(L7AT*2u2Gvvo#9q;LzmAh8Q{vGtO#n5h6G`t}e&(Klft?qlFGjI`%5_UOFAO z7!!h;e6Td#nAw-`G+ab~Z`{FP;jg^*bKa=1BQqsyt6JZmuj|&rSP7o-WTH%O^!><< z2(iur26)t)f6q{7m|FViy`_m*eXkHlO*DZH#68MIliQQK|@ghc`tm8K;}8_ zN*T^ywSA(&_oH{Y;S)+-;_sfZrPfUzmYTZ`ES<*W=oPLjx+!&?}2VK)XSkwX7-Pl&=f04_#;Ft9rB4 zZB`cQH0pt^xoMZ+ZVzxoJ@<;Nh#~B`!3O67+*)M^#F97HSDm4Uh3Ma-YZF6|O&K zX$4Htp031yp9J^(%CqIbvK3!u?a2ZwYV9XQWS9pCJ75<``_5?)=n4sS&D`D(XrFjJn2*^YM#PiYH z=FaAmQ0xBxX(#l|nzLbIl#fl_c;ChpRUzl;n(niuYK7S}UrfVWKkx^1EF}6)B(jJn z0*^StO*7=I-$L}bC-@h{!Gu==b8ppvKrb5_ZH|lWZvp+J;J%xXh?VksB}TA93{G^N z1(<<#vnMk^+$)xsMj-{A49;67kOnon4YRM~%J{w3z?Dp!w}3xPZ&cW+(Dxh8mG*ek zBuqwqDjLuR)z@sfhVEbg_FCFqY+)U;+$E5iZu9kRK6LmU#RPAI{%;#*Zmv|Pb(z~j z^&43ziI9Hxlv)@KPwQbupH>gE-Nn*ys^IO0YrYNQf??^=1=?@eV>TECnQc0KlHC}eD zHxy(GMa;PqJOMgd^#_3K*m8iy8iVc&`uL&VEyC1Qu9eB!&r{#m(<3CFx|yMI0lx{Z zT?2UqF1n2K09v=7%E19&Ts3smQV^O%!QT$Vm5``0LE8EWqj2ypCt|q9TMD5n&&R&K z*lp2dA6N#8exkTj3f`3lzj@(V^?TCxak`=l}HE>5{;M{a_TU`$!?WosaA~sMx>eDNZ#CpCks{ zN6O0d76G)tl2QC|8dz_yz2umjHoreZ&`>uIFvPFL&~McgctT zS6ltXi2SYOy&n9O-#cKM2%WDnD%fm~-|eJsM5&t|Pp|M!od7eA`NS!K5{a z><=lT*=9N(yDzK)9DU1FFW!KwwAkBG0gYyS6C}iFzkfG2Yb%^52|=m(GuH9*YBxda3EL)*jB(#& z3gc7&5MaWD_OKr9zajn+pf`Cz>UilX{rqGmmEaBjUmSlxtGPch879-8DKakp zLz3vwCVpymTIEr;fwGp;5SCv{kfYw4%zIt~(Z>~82c z{{vNSa*lz#`|(4c_xWdXBj@L+{EIsh_&bTHbh{j4!P8Pi))B&$0rDlL-q2zVF5u>IE6ebNNi?{^xOep$Er07*%n(@L67e4Duf!(XDJ14^i5P2*#$mK6m@SS>m z^=mdG!pY@uxhVgNE)1Ie_Wy0{; zDINCFgds<2Yfldu{LFO8pwua!cQ1=I_QhIY`vZLh73C)`Uj5{-YM2$Oz3KR+^2ul7 z4gn)64}b*cxcL=+X}bkiBxk}%hFQqFe3Dy4M6_4*k(!vrP+`wY-i8yWN#yi=nZziQCcnFhfGCQcSGQN~#sf=eHhUtY>$VMeiEb6-oQ{n>4$A zKC@4GpUK!B04?Dx($(eg+0F@JXjHhrYZYizPW}qBy8_hxqo20XsMt6M-`|L)Dxc0D zC%8M1y#w6|>Jot6!NCUXg*m<@{sbs_r;iQLZV1%iRMx5|E(9WV{SN*yJL} zFn#$6_NP{dg2&A8p=-SdgQ9gp9odbP9R!_M078D5OdhCL5|ro5>X|Jy&M|1{Sy-T1}!r*KntBaOd@OIM56ITn$bc3DYT zUL&^6{A=T%r+-){TR%Tu*t^WfP}wWJmtm7DD=W~yv3!uS;DR~_({mP#uY@6KmRp1v zTS&&CmJ`G0-e88)LwIIUAD`f4%b28cm?$ygw;d!K3VThVU+Yl~QO7_-Uy!|G3G+s>WgMEBr$zu4o zA*3}zvgOnV6~k~Bx0qtxQYrGYiZlgHTiT>Ox=V3!xJDomCXEbFBYQpDKh&7?=ZH>V zQiQUJ$tmSWFmF@coK|*>q=?rN)X<+Xio+E-LE_2^d4dSo|4=;91~%R2;hQK-4Q+GK zMZvJytW*+mt41BE!w=NPUFdsF^^eg{yyOEeDW5;o|D+pkv1s7<`K=tseXrOMfw&@6 zKd#&{pU>^)gZrAUOZzPWyAw7yVijNQh%nZO!sD`S(c=ut(WW#CL_ikJpaXn#6&G`Z!wz|?*!0vJayWj*?B}Q zyUJZBptNh82h@1Afq96@0TrMs;y5v%0S2z7)kY#`V1p*ptgPF-%hx!i9WS!RNqqDn zy>U*DaN@c$e*455E@o-M9EqxDWUwqf2AwTkwqZs|lMgSjp0c3mFNrq7?!>QTd6Q^d z=&$l#H=7hXNnuHC_eh*%Q{lYmZ^gP|1&K|{*)a*N-6!6++hptP>a~bu+j7KxI34SK zc~*lbt1KwN8pM;WCuhEC_@DmB*kL6f41v``&)6#?8e(LWJU+mR(n(3a_eEHYwyUaZ zxONtxpQVQD@p9C(64cBPqKNp(4PlNzH{gyvcX+--W!oV9+oyw?VP+v>wO*6$#(#uz6*Tr)u?I!}B?R_OEz(({k{o~`DaJ}>+koVC9d4scp%GuHG zWF#z{P1GX}B}h-Zt*C#xc0K&dOcRftt{EBP`dv)ZIGt(FLFpOJQ`LD6(kL*#h~aou z@6TQ~Ds__5{z>L2#NpbmfO@T$|8>W8?jy>%?4K4L6jzqzT>7u`qP%D-4ka^ecm=D# ziQjnjR1_mu^$(fM^e$YY9Akrvis93@6rtyVSKSp$&7QTJ(MwbLyk7l9m|I;hu3tf@ zf*yt@=cAzs7N8Q>S69Tm&(^G#im)>#w#awSoz8zm@ov`qg20k~Oiv9NBnAa)h zW|oxn&xTINZDXU}l1T&^VEFG6>z-@ z{;hGe>bE648%|fjWFklT<4w6v*6o$uvF;l#qXGBlMIYU$+mF= z59p9zbPGS_MOJ3h6uszLaw#@*K@9drbGK*de{+#7IEUOO5G&FH?4F1NlLPkQQNf$b z!qm?dzO2HX2YU!4X5W3tEkqn++0sR`>g%-uLt*nHgk0XZht zNkJ`ULn<;y#98tfK_#oq++6${h=B4|%m7!k(|4PoB;az_SB|V%E(M?aF&mwe%CGK4 z9^1IK%?a7wP~sn>0L4~w#IcWa7pTP0VV~bpW*>n_M{~`rU3#SR zr_+oUNY*u%A|XuTc6U5Q(ui?6xf)!Tc-1HdM5|;0;OP_kDM9R>#S(cr;#lqm-pYBb zTNdASObE~qH_`x#d|vP=>9I!W?=tM@)wDCnrO@OlDrh{%#NhM=Ai3D$VVS1`S%q)(4na5E+{&wVh7E>! z57M9_Q0`fG@6u$OxgUQG*`YzMcvLIl(})ujNaDV8M8H=`0BMQnIK%NJjUYfzi75+Y zD|Q5~Fqqs`sSMMz-oFzMxr`vLW0vlM!oA>725B21t6B~SLB;|%Yt>Xj_9J9rs%~m~ zcoSvIetLK|lQ|r^LDsG&5+12W^Hv$A)z=YawhxOgVsfLNL->ddYsn1_4OuKGS?)u_ z4F&=>#fuIyeHN#f{+yqz{dLG25djZRJCZ;uz`sN?=Q>E7a;sc5frU9JEdu)OoxB=r yLKY>0803~LB9wh*YK~Gk5pjfG{eS6b=OM}6UEc;>X%%K7_|r$3AZxW8BL5$E@#+8o literal 0 HcmV?d00001 diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ new file mode 100644 index 000000000..0052389ba --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ @@ -0,0 +1,209 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: custom-metrics +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: custom-metrics:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: custom-metrics-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: custom-metrics-resource-reader + namespace: ${namespace} +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: custom-metrics-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: custom-metrics-resource-reader +subjects: +- kind: ServiceAccount + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} + labels: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter +spec: + replicas: 1 + selector: + matchLabels: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter + template: + metadata: + labels: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: custom-metrics-stackdriver-adapter + priorityClassName: higher-priority + containers: + - image: gcr.io/gke-release/custom-metrics-stackdriver-adapter:v0.15.1-gke.0 + imagePullPolicy: Always + name: pod-custom-metrics-stackdriver-adapter + command: + - /adapter + - --use-new-resource-model=true + - --fallback-for-container-metrics=true + resources: + limits: + cpu: 250m + memory: 200Mi + requests: + cpu: 250m + memory: 200Mi +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter + kubernetes.io/cluster-service: 'true' + kubernetes.io/name: Adapter + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter + type: ClusterIP +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + insecureSkipTLSVerify: true + group: custom.metrics.k8s.io + groupPriorityMinimum: 100 + versionPriority: 100 + service: + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} + version: v1beta1 +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta2.custom.metrics.k8s.io +spec: + insecureSkipTLSVerify: true + group: custom.metrics.k8s.io + groupPriorityMinimum: 100 + versionPriority: 200 + service: + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} + version: v1beta2 +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.external.metrics.k8s.io +spec: + insecureSkipTLSVerify: true + group: external.metrics.k8s.io + groupPriorityMinimum: 100 + versionPriority: 100 + service: + name: custom-metrics-stackdriver-adapter + namespace: ${namespace} + version: v1beta1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: external-metrics-reader +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: external-metrics-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-metrics-reader +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/agent_job.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/agent_job.templ new file mode 100644 index 000000000..a4f47e41b --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/agent_job.templ @@ -0,0 +1,107 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: ${name} + namespace: ${namespace} +spec: + parallelism: ${parallel} + ttlSecondsAfterFinished: 30 + template: + metadata: + labels: + app: ${name} + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/cpu-limit: "1" + gke-gcsfuse/memory-limit: "1Gi" + gke-gcsfuse/ephemeral-storage-request: "1Gi" + spec: + priorityClassName: higher-priority + nodeSelector: + cloud.google.com/compute-class: spot-capacity + initContainers: + - name: workload + image: ${workload_image} + args: ${jsonencode(workload_args)} + restartPolicy: Always + workingDir: /data + resources: + requests: + memory: "2Gi" + cpu: "1" + limits: + cpu: "2" + memory: "4Gi" + volumeMounts: + - name: gcs-data + mountPath: /data + containers: + - name: agent + image: ${agent_image} + resources: + requests: + memory: "2Gi" + cpu: "1" + limits: + cpu: "2" + memory: "4Gi" + args: ["serve", "pubsub-pull", + # subscription to pull requests (tasks) from, topic to publish responses + "${workload_request_sub}", "${workload_response}", + # Log to JSON + "--logJSON", + # Log every operation + "--logAll", + # Text encode protobuf on PubSub + "--jsonPubSub=true", + # Endpoint to dispatch the work + "--endpoint", "${workload_endpoint}", + # Timeout waiting for the gRPC service to be available + "--timeout", "30s", + # Timeout when there's no more work from Pub/Sub -- stop everything. + "--idleTimeout", "120s", + # NOTE: These are important so that only a small number of messages (tasks) + # are pulled from Pub/Sub, as if they are chunky in size they need to be + # evenly distributed. + # Maximum number of goroutines executing + "--goroutines", "1", + # Maximum number of outstanding messages + "--maxoutstandingmessages", "1"] + env: + - name: PUBSUB_PROJECT_ID + value: "${pubsub_project_id}" + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: Never + volumes: + - name: gcs-data + persistentVolumeClaim: + claimName: data-pvc + readOnly: false + + backoffLimit: 2 + podFailurePolicy: + rules: + - action: FailJob + onExitCodes: + containerName: agent + operator: NotIn + values: [1] diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/controller_job.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/controller_job.templ new file mode 100644 index 000000000..04aba4075 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/controller_job.templ @@ -0,0 +1,64 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: ${job_name} + namespace: ${namespace} +spec: + parallelism: ${parallel} + ttlSecondsAfterFinished: 30 + backoffLimit: 2 + template: + metadata: + labels: + app: ${job_name} + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/cpu-limit: "1" + gke-gcsfuse/memory-limit: "1Gi" + gke-gcsfuse/ephemeral-storage-request: "1Gi" + spec: + priorityClassName: higher-priority + containers: + - name: ${container_name} + image: ${image} + workingDir: /data + args: ${jsonencode(args)} + resources: + requests: + memory: "2Gi" + cpu: "2" + limits: + cpu: "4" + memory: "4Gi" + volumeMounts: + - name: gcs-data + mountPath: /data + env: + - name: PUBSUB_PROJECT_ID + value: "${pubsub_project_id}" + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: Never + volumes: + - name: gcs-data + persistentVolumeClaim: + claimName: data-pvc + readOnly: false diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ new file mode 100644 index 000000000..72e03c750 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ @@ -0,0 +1,111 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: ${name} + namespace: ${namespace} +spec: + minReplicas: 1 + maxReplicas: 2 + metrics: + - external: + metric: + name: pubsub.googleapis.com|subscription|num_undelivered_messages + selector: + matchLabels: + resource.labels.subscription_id: "${gke_hpa_request_sub}" + resource.type: "pubsub_subscription" + resource.labels.project_id: "${pubsub_project_id}" + target: + type: AverageValue + averageValue: 1 + type: External + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: ${name} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${name} + namespace: ${namespace} + labels: + app: ${name} +spec: + replicas: 1 + selector: + matchLabels: + app: ${name} + template: + metadata: + labels: + app: ${name} + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/cpu-limit: "1" + gke-gcsfuse/memory-limit: "1Gi" + gke-gcsfuse/ephemeral-storage-request: "1Gi" + spec: + nodeSelector: + cloud.google.com/compute-class: spot-capacity + containers: + - name: workload + image: ${workload_image} + args: ${jsonencode(workload_args)} + workingDir: /data + resources: + requests: + memory: "1Gi" + cpu: "200m" + limits: + cpu: "250m" + memory: "1Gi" + volumeMounts: + - name: gcs-data + mountPath: /data + - name: agent + resources: + requests: + memory: "1Gi" + cpu: "900m" + limits: + cpu: "1000m" + memory: "3Gi" + image: ${agent_image} + args: ["serve", "pubsub-pull", + "${gke_hpa_request_sub}", "${gke_hpa_response}", + "--logJSON", + "--logAll", + "--jsonPubSub=true", + "--endpoint", "${workload_endpoint}", + "--timeout", "120s", + "--goroutines", "1", + "--maxoutstandingmessages", "1"] + env: + - name: PUBSUB_PROJECT_ID + value: "${pubsub_project_id}" + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: gcs-data + persistentVolumeClaim: + claimName: data-pvc + readOnly: false diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/job.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/job.templ new file mode 100644 index 000000000..c0c29738e --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/job.templ @@ -0,0 +1,66 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: ${job_name} + namespace: ${namespace} +spec: + parallelism: ${parallel} + ttlSecondsAfterFinished: 30 + backoffLimit: 2 + template: + metadata: + labels: + app: ${job_name} + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/cpu-limit: "1" + gke-gcsfuse/memory-limit: "1Gi" + gke-gcsfuse/cpu-limit: "1" + gke-gcsfuse/memory-limit: "1Gi" + gke-gcsfuse/ephemeral-storage-request: "1Gi" + spec: + priorityClassName: higher-priority + containers: + - name: ${container_name} + image: ${image} + workingDir: /data + args: ${jsonencode(args)} + resources: + requests: + memory: "1Gi" + cpu: "900m" + limits: + cpu: "1000m" + memory: "2Gi" + volumeMounts: + - name: gcs-data + mountPath: /data + env: + - name: PUBSUB_PROJECT_ID + value: "${pubsub_project_id}" + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: Never + volumes: + - name: gcs-data + persistentVolumeClaim: + claimName: data-pvc + readOnly: false diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/agent_job.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/agent_job.templ new file mode 100644 index 000000000..6643eca13 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/agent_job.templ @@ -0,0 +1,104 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: ${name} +spec: + parallelism: ${parallel} + ttlSecondsAfterFinished: 30 + template: + metadata: + labels: + app: ${name} + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/cpu-limit: "1" + gke-gcsfuse/memory-limit: "1Gi" + gke-gcsfuse/ephemeral-storage-request: "1Gi" + spec: + priorityClassName: higher-priority + nodeSelector: + cloud.google.com/compute-class: spot-capacity + initContainers: + - name: workload + image: ${workload_image} + args: ${jsonencode(workload_args)} + restartPolicy: Always + workingDir: /data + resources: + requests: + memory: "2Gi" + cpu: "1" + limits: + cpu: "2" + memory: "4Gi" + volumeMounts: + - name: gcs-data + mountPath: /data + containers: + - name: agent + image: ${agent_image} + resources: + requests: + memory: "2Gi" + cpu: "1" + limits: + cpu: "2" + memory: "4Gi" + args: ["serve", "pubsub-pull", + # subscription to pull requests (tasks) from, topic to publish responses + "${workload_request_sub}", "${workload_response}", + # Log to JSON + "--logJSON", + # Log every operation + "--logAll", + # Text encode protobuf on PubSub + "--jsonPubSub=true", + # Endpoint to dispatch the work + "--endpoint", "${workload_endpoint}", + # Timeout waiting for the gRPC service to be available + "--timeout", "30s", + # Timeout when there's no more work from Pub/Sub -- stop everything. + "--idleTimeout", "120s", + # NOTE: These are important so that only a small number of messages (tasks) + # are pulled from Pub/Sub, as if they are chunky in size they need to be + # evenly distributed. + # Maximum number of goroutines executing + "--goroutines", "1", + # Maximum number of outstanding messages + "--maxoutstandingmessages", "1"] + env: + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: Never + volumes: + - name: gcs-data + persistentVolumeClaim: + claimName: data-pvc + readOnly: false + + backoffLimit: 2 + podFailurePolicy: + rules: + - action: FailJob + onExitCodes: + containerName: agent + operator: NotIn + values: [1] diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/controller_job.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/controller_job.templ new file mode 100644 index 000000000..a5598dfe6 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/controller_job.templ @@ -0,0 +1,53 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: ${job_name} +spec: + parallelism: ${parallel} + ttlSecondsAfterFinished: 30 + backoffLimit: 2 + template: + metadata: + labels: + app: ${job_name} + spec: + priorityClassName: higher-priority + containers: + - name: ${container_name} + image: ${image} + workingDir: /data + args: ${jsonencode(args)} + resources: + requests: + memory: "2Gi" + cpu: "2" + limits: + cpu: "4" + memory: "4Gi" + volumeMounts: + - name: parallelstore-volume + mountPath: /data + env: + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: Never + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc-${location} diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/hpa.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/hpa.yaml.templ new file mode 100644 index 000000000..e084f3d53 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/hpa.yaml.templ @@ -0,0 +1,118 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: ${name} +spec: + minReplicas: 1 + maxReplicas: 50000 + metrics: + - external: + metric: + name: pubsub.googleapis.com|subscription|num_undelivered_messages + selector: + matchLabels: + resource.labels.subscription_id: ${gke_hpa_request_sub} + target: + type: AverageValue + averageValue: 1 + type: External + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: ${name} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${name} + labels: + app: ${name} +spec: + replicas: 1 + selector: + matchLabels: + app: ${name} + template: + metadata: + labels: + app: ${name} + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/cpu-limit: "1" + gke-gcsfuse/memory-limit: "1Gi" + gke-gcsfuse/ephemeral-storage-request: "1Gi" + spec: + nodeSelector: + cloud.google.com/compute-class: spot-capacity + containers: + - name: workload + # Workload. It starts and waits on port 2002 with gRPC service. + image: ${workload_image} + args: ${jsonencode(workload_args)} + # Tasks may require reading/writing to a POSIX file. /data is mapped + # for this purpose. + workingDir: /data + resources: + requests: + memory: "1Gi" + cpu: "200m" + limits: + cpu: "200m" + memory: "1Gi" + volumeMounts: + - name: parallelstore-volume + mountPath: /data + - name: agent + resources: + requests: + memory: "3Gi" + cpu: "800m" + limits: + cpu: "800m" + memory: "3Gi" + image: ${agent_image} + args: ["serve", "pubsub-pull", + # subscription to pull requests (tasks) from, topic to publish responses + "${gke_hpa_request_sub}", "${gke_hpa_response}", + # Log to JSON + "--logJSON", + # Log Every operation + "--logAll", + # Text encode protobuf on PubSub + "--jsonPubSub=true", + # Endpoint to dispatch the work + "--endpoint", "${workload_endpoint}", + # Timeout waiting for the gRPC service to be available + "--timeout", "120s", + # NOTE: These are important so that only a small number of messages (tasks) + # are pulled from Pub/Sub, as if they are chunky in size they need to be + # evenly distributed. + # Maximum number of goroutines executing + "--goroutines", "1", + # Maximum number of outstanding messages + "--maxoutstandingmessages", "1"] + env: + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc-${location} diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/job.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/job.templ new file mode 100644 index 000000000..8a8531b3a --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/job.templ @@ -0,0 +1,55 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: ${job_name} +spec: + parallelism: ${parallel} + ttlSecondsAfterFinished: 30 + backoffLimit: 5 + template: + metadata: + labels: + app: ${job_name} + spec: + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: higher-priority + containers: + - name: ${container_name} + image: ${image} + workingDir: /data + args: ${jsonencode(args)} + resources: + requests: + memory: "1Gi" + cpu: "900m" + limits: + cpu: "2" + memory: "2Gi" + volumeMounts: + - name: parallelstore-volume + mountPath: /data + env: + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: Never + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc-${location} diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/volume.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/volume.yaml.templ new file mode 100644 index 000000000..c516e9365 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/volume.yaml.templ @@ -0,0 +1,37 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolume +metadata: + name: parallelstore-pv-${location} +spec: + storageClassName: "parallelstore-sc-${location}" + capacity: + storage: ${capacity}Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + csi: + driver: parallelstore.csi.storage.gke.io + volumeHandle: "${project_id}/${location}/${instance_name}/default-pool/default-container" + volumeAttributes: + accessPoints: ${access_points} + network: ${vpc} + mountLocality: node + dfuseCPURequest: "4" + dfuseMemoryRequest: "3Gi" + dfuseCPULimit: "6" + dfuseMemoryLimit: "5Gi" diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/volume_claim.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/volume_claim.yaml.templ new file mode 100644 index 000000000..d64a47c87 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/parallelstore/volume_claim.yaml.templ @@ -0,0 +1,25 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: parallelstore-pvc-${location} +spec: + accessModes: + - ReadWriteMany + storageClassName: "parallelstore-sc-${location}" + resources: + requests: + storage: ${capacity}Gi diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/test_config.sh.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/test_config.sh.templ new file mode 100644 index 000000000..1ab123f8b --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/test_config.sh.templ @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +set -x + +export KUBECONFIG="/tmp/kubeconfig_${cluster_name}_${project_id}" +if [ ! -r "$KUBECONFIG" ]; then + TEMP_KUBECONFIG=$(mktemp "/tmp/kubeconfig_XXXXXX") + KUBECONFIG="$TEMP_KUBECONFIG" gcloud container fleet memberships get-credentials ${cluster_name} --project=${project_id} --location=${region} + mv -f "$TEMP_KUBECONFIG" "$KUBECONFIG" +fi + +if [ "${parallel}" -ne "0" ]; then +kubectl apply --server-side -f - < { + args = args, + image = var.workload_image, + } + } + + parallelstore_templates = var.parallelstore_enabled ? fileset("${path.module}/k8s/parallelstore", "*.yaml.templ") : [] + + parallelstore_configs = { for fname in local.parallelstore_templates : fname => { + name = "parallelstore-${replace(fname, ".yaml.templ", "")}" + template_path = "${path.module}/k8s/parallelstore/${fname}" + } } + + cluster_config = "${var.cluster_name}-${var.region}-${var.cluster_project_id}" + + kubeconfig_script = "gcloud container fleet memberships get-credentials ${var.cluster_name} --project ${var.cluster_project_id} --location ${var.region}" + + + # Test output + test_job_template = { + for id, cfg in var.test_configs : + id => templatefile( + "${path.module}/k8s/agent_job.templ", { + name = "${replace(id, "/[_\\.]/", "-")}-worker", + parallel = cfg.parallel, + workload_args = var.workload_args, + workload_image = var.workload_image, + namespace = var.namespace, + pubsub_project_id = var.infra_project_id, + agent_image = var.agent_image, + workload_endpoint = var.workload_grpc_endpoint, + workload_request_sub = (cfg.parallel > 0 ? + var.pubsub_job_request : + var.pubsub_hpa_request) + workload_response = (cfg.parallel > 0 ? + var.gke_job_response : var.gke_hpa_response) + }) + } + + test_controller_template = { + for id, cfg in var.test_configs : + id => templatefile( + "${path.module}/k8s/controller_job.templ", { + parallel = 1, + job_name = "${replace(id, "/[_\\.]/", "-")}-controller", + container_name = "controller", + namespace = var.namespace, + image = var.agent_image, + pubsub_project_id = var.infra_project_id, + args = [ + "test", "pubsub", + "--logJSON", + "--logAll", + "--jsonPubSub=true", + (cfg.parallel > 0 ? + var.gke_job_request : var.gke_hpa_request), + (cfg.parallel > 0 ? + var.pubsub_job_request : + var.pubsub_hpa_request), + "--source", + cfg.testfile] + }) + } + + test_shell = { + for id, cfg in var.test_configs : + id => templatefile( + "${path.module}/k8s/test_config.sh.templ", { + namespace = var.namespace, + pubsub_project_id = var.infra_project_id, + parallel = cfg.parallel, + job_config = local.test_job_template[id], + controller_config = local.test_controller_template[id], + project_id = var.cluster_project_id, + region = var.region, + cluster_name = var.cluster_name, + KUBECONFIG = "/tmp/kubeconfig_${var.cluster_name}-${var.cluster_project_id}.yaml" + }) + } +} + +# Apply configurations to the cluster +resource "null_resource" "cluster_init" { + for_each = merge( + { for fname in fileset(".", "${path.module}/k8s/*.yaml") : fname => file(fname) }, + { "volume_yaml" = templatefile( + "${path.module}/k8s/volume.yaml.templ", { + gcs_storage_data = var.gcs_bucket + namespace = var.namespace + }), + "hpa_yaml" = templatefile( + "${path.module}/k8s/hpa.yaml.templ", { + name = "gke-hpa" + namespace = var.namespace + pubsub_project_id = var.infra_project_id + workload_image = var.workload_image + workload_args = var.workload_args + workload_endpoint = var.workload_grpc_endpoint + agent_image = var.agent_image + gke_hpa_request_sub = var.pubsub_hpa_request + gke_hpa_response = var.gke_hpa_response + }), + "adapter_new_resource_model_yaml" = templatefile( + "${path.module}/k8s/adapter_new_resource_model.yaml.templ", { + namespace = var.namespace + }), + "volume_claim_yaml" = templatefile( + "${path.module}/k8s/volume_claim.yaml.templ", { + namespace = var.namespace + }) + } + ) + + triggers = { + template = each.value + cluster_change = local.cluster_config + test = true + } + + provisioner "local-exec" { + when = create + command = <<-EOT + ${local.kubeconfig_script} + + mkdir -p ./generated/k8s_configs + echo "${each.value}" ./generated/k8s_configs/${each.key} + + kubectl apply -f - < templatefile("${path.module}/k8s/job.templ", { + job_name = replace(id, "/[_\\.]/", "-"), + container_name = replace(id, "/[_\\.]/", "-"), + parallel = 1, + image = cfg.image, + args = cfg.args, + namespace = var.namespace + pubsub_project_id = var.infra_project_id + }) + } + + depends_on = [null_resource.cluster_init] + + triggers = { + cluster_change = local.cluster_config + } + + provisioner "local-exec" { + when = create + command = <<-EOT + ${local.kubeconfig_script} + + mkdir -p ${path.module}/../../../generated/k8s_configs/job_init + echo "${each.value}" > ${path.module}/../../../generated/k8s_configs/job_init/${each.key}.yaml + + kubectl apply --v=6 -f - < ${path.module}/../../../generated/k8s_configs/parallelstore_init/${each.key} + kubectl apply -f - < templatefile("${path.module}/k8s/parallelstore/job.templ", { + job_name = "parallelstore-${replace(id, "/[_\\.]/", "-")}" + container_name = replace(id, "/[_\\.]/", "-") + access_points = var.parallelstore_access_points + cluster_project_id = var.cluster_project_id + vpc = var.parallelstore_vpc_name + location = var.parallelstore_location + instance_name = var.parallelstore_instance_name + capacity = var.parallelstore_capacity_gib + namespace = var.namespace + pubsub_project_id = var.infra_project_id + parallel = 1 + image = cfg.image + args = cfg.args + }) + } : {} + + depends_on = [null_resource.parallelstore_init] + + triggers = { + cluster_change = local.cluster_config + } + + provisioner "local-exec" { + when = create + command = <<-EOT + ${local.kubeconfig_script} + mkdir -p ${path.module}/../../../generated/k8s_configs/parallelstore_job_init + echo "${each.value}" > ${path.module}/../../../generated/k8s_configs/parallelstore_job_init/${each.key}.yaml + kubectl apply -f - < bool 60)", + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "STACKED_BAR", + "legendTemplate": "", + "targetAxis": "Y2", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "prometheusQuery": "sum(increase(kubernetes_io:container_restart_count{monitored_resource=\"k8s_container\",pod_name=~\"gke-hpa.+\"}[${__interval}]))", + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "STACKED_BAR", + "legendTemplate": "", + "targetAxis": "Y2", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "y2Axis": { + "label": "Count", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "id": "" + } + }, + { + "yPos": 48, + "width": 48, + "height": 48, + "widget": { + "title": "GKE", + "collapsibleGroup": { + "collapsed": true + }, + "id": "" + } + }, + { + "xPos": 24, + "yPos": 48, + "width": 24, + "height": 16, + "widget": { + "title": "Number of instances per machine type", + "timeSeriesTable": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_COUNT", + "groupByFields": [ + "metadata.system_labels.\"machine_type\"" + ] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "tableTemplate": "", + "minAlignmentPeriod": "60s" + } + ], + "metricVisualization": "NUMBER", + "columnSettings": [ + { + "column": "system_labels.machine_type", + "visible": true, + "displayName": "Machine Type" + }, + { + "column": "project_id", + "visible": false + }, + { + "column": "value", + "visible": true + } + ], + "opsAnalyticsSettings": { + "maxRows": "0", + "showFilterBar": false, + "pageSize": "0" + }, + "displayColumnType": false + }, + "id": "" + } + }, + { + "yPos": 64, + "width": 24, + "height": 16, + "widget": { + "title": "CPU - total usage / request / limit", + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/cpu/core_usage_time\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_RATE", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/cpu/limit_cores\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/cpu/request_cores\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "id": "" + } + }, + { + "xPos": 24, + "yPos": 64, + "width": 24, + "height": 16, + "widget": { + "title": "Memory - total usage / request / limit", + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/memory/used_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y2", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/memory/limit_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y2", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/memory/request_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y2", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "y2Axis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "id": "" + } + }, + { + "yPos": 24, + "width": 48, + "height": 24, + "widget": { + "title": "Worker", + "collapsibleGroup": { + "collapsed": true + }, + "id": "" + } + }, + { + "yPos": 96, + "width": 48, + "height": 32, + "widget": { + "title": "Pub / Sub", + "collapsibleGroup": { + "collapsed": true + }, + "id": "" + } + }, + { + "xPos": 24, + "yPos": 80, + "width": 24, + "height": 16, + "widget": { + "title": "Autoscaler events", + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"logging.googleapis.com/log_entry_count\" resource.type=\"k8s_cluster\" metric.label.\"log\"=\"container.googleapis.com/cluster-autoscaler-visibility\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_RATE", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "STACKED_BAR", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "id": "" + } + }, + { + "yPos": 80, + "width": 24, + "height": 16, + "widget": { + "title": "Ephemeral Storage - total usage / request / limit", + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/ephemeral_storage/used_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y2", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/ephemeral_storage/limit_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y2", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"kubernetes.io/container/ephemeral_storage/request_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "minAlignmentPeriod": "60s", + "targetAxis": "Y2", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "y2Axis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "id": "" + } + }, + { + "yPos": 24, + "width": 24, + "height": 24, + "widget": { + "title": "Worker replica stats", + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/kube_horizontalpodautoscaler_spec_min_replicas/gauge\" resource.type=\"prometheus_target\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + }, + "pickTimeSeriesFilter": { + "rankingMethod": "METHOD_MEAN", + "numTimeSeries": 30, + "direction": "TOP" + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "Min Replicas", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/kube_horizontalpodautoscaler_spec_max_replicas/gauge\" resource.type=\"prometheus_target\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + }, + "pickTimeSeriesFilter": { + "rankingMethod": "METHOD_MEAN", + "numTimeSeries": 30, + "direction": "TOP" + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "Max Replicas", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/kube_horizontalpodautoscaler_status_desired_replicas/gauge\" resource.type=\"prometheus_target\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + }, + "pickTimeSeriesFilter": { + "rankingMethod": "METHOD_MEAN", + "numTimeSeries": 30, + "direction": "TOP" + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "Desired Replicas", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/kube_horizontalpodautoscaler_status_current_replicas/gauge\" resource.type=\"prometheus_target\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + }, + "pickTimeSeriesFilter": { + "rankingMethod": "METHOD_MEAN", + "numTimeSeries": 30, + "direction": "TOP" + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "Current Replicas", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "id": "" + } + } + ] + }, + "dashboardFilters": [], + "labels": {} + } diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf new file mode 100644 index 000000000..98a4874aa --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf @@ -0,0 +1,125 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_project" "environment" { + project_id = var.project_id +} + +locals { + enable_jobs = (var.gke_job_request != "" && var.gke_job_response != "") ? 1 : 0 + enable_hpa = (var.gke_hpa_request != "" && var.gke_job_response != "") ? 1 : 0 + # Topics + pubsub_topics = concat( + local.enable_jobs == 1 ? [ + var.gke_job_request, + var.gke_job_response, + ] : [], + local.enable_hpa == 1 ? [ + var.gke_hpa_request, + var.gke_hpa_response, + ] : [], + ) + clusters = [for s in var.gke_cluster_names : { + name = s + location = try(regex("cluster-([a-z0-9-]+)-${var.env}", s)[0], null) + }] +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +# Global Resources + +# Pubsub +resource "google_pubsub_topic" "topic" { + for_each = toset(local.pubsub_topics) + project = var.project_id + name = each.value + message_storage_policy { + allowed_persistence_regions = var.regions + } +} + +resource "google_pubsub_subscription" "subscription" { + for_each = toset(local.pubsub_topics) + project = google_pubsub_topic.topic[each.value].project + topic = google_pubsub_topic.topic[each.value].name + name = "${each.value}_sub" + enable_exactly_once_delivery = var.pubsub_exactly_once + ack_deadline_seconds = 60 + retry_policy { + minimum_backoff = "30s" + maximum_backoff = "600s" + } +} + +# Dashboard + +resource "google_monitoring_dashboard" "risk-platform-overview" { + project = data.google_project.environment.project_id + dashboard_json = file("${path.module}/${var.dashboard}") + + lifecycle { + ignore_changes = [ + dashboard_json + ] + } +} +# Regional Resources + +# GCS bucket per region +resource "google_storage_bucket" "gcs_storage_data" { + for_each = toset(var.regions) + project = var.project_id + location = each.value + name = "${var.project_id}-${each.value}-gke-data-${random_string.suffix.id}" + uniform_bucket_level_access = true + force_destroy = true + hierarchical_namespace { + enabled = var.hsn_bucket + } +} + +# Apply Resource to each GKE cluster +module "config_apply" { + for_each = { for idx, cluster in local.clusters : idx => cluster } + source = "../config_apply" + cluster_project_id = var.cluster_project_id + infra_project_id = var.project_id + region = each.value.location + cluster_name = each.value.name + agent_image = var.agent_image + namespace = var.namespace + # Workload options + workload_image = var.workload_image + workload_args = var.workload_args + workload_grpc_endpoint = var.workload_grpc_endpoint + workload_init_args = var.workload_init_args + test_configs = var.test_configs + gcs_bucket = google_storage_bucket.gcs_storage_data[each.value.location].id + # set this to empty string if not enabled + pubsub_hpa_request = local.enable_hpa == 1 ? google_pubsub_subscription.subscription[var.gke_hpa_request].name : "" + pubsub_job_request = local.enable_jobs == 1 ? google_pubsub_subscription.subscription[var.gke_job_request].name : "" + + # Parallelstore Config + parallelstore_enabled = var.parallelstore_enabled + parallelstore_access_points = var.parallelstore_enabled ? join(",", var.parallelstore_instances[each.value.location].access_points) : null + parallelstore_vpc_name = var.parallelstore_enabled ? var.vpc_name : null + parallelstore_location = var.parallelstore_enabled ? var.parallelstore_instances[each.value.location].location : null + parallelstore_instance_name = var.parallelstore_enabled ? var.parallelstore_instances[each.value.location].name : null + parallelstore_capacity_gib = var.parallelstore_enabled ? var.parallelstore_instances[each.value.location].capacity_gib : null +} diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf new file mode 100644 index 000000000..f2fd2e11d --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf @@ -0,0 +1,53 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Topics (for BigQuery capturing of output) +output "topics" { + description = "Image and tag names for build containers" + value = local.pubsub_topics + depends_on = [ + google_pubsub_topic.topic, + google_pubsub_subscription.subscription + ] +} + +# Dashboard for Platform Overview +output "monitoring_dashboard_url" { + description = "Cloud Monitoring dashboard" + value = "https://console.cloud.google.com/monitoring/dashboards/builder/${regex("projects/[0-9]+/dashboards/(.*)$", google_monitoring_dashboard.risk-platform-overview.id)[0]};project=${var.project_id}" +} + +# Test scripts (shell scripts) +output "test_scripts_list" { + description = "Test configuration shell scripts as a list" + value = flatten([ + for cluster_module in module.config_apply : [ + for script_id, script in cluster_module.test_scripts : script + ] + ]) +} + +output "first_test_script" { + description = "First test script" + value = module.config_apply[0].test_scripts +} + +# Cluster +output "cluster_urls" { + description = "Cluster urls" + value = { + for idx, cluster in local.clusters : + cluster.name => "https://console.cloud.google.com/kubernetes/workload/overview?project=${var.project_id}&pageState=(%22savedViews%22:(%22n%22:%5B%22default%22%5D,%22c%22:%5B%22gke%2F${cluster.location}%2F${cluster.name}%22%5D))" + } +} diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf new file mode 100644 index 000000000..d6acc1469 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf @@ -0,0 +1,176 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# Mandatory configuration +# + +# Project ID where resources will be deployed +variable "project_id" { + type = string + description = "The GCP project ID where resources will be created." +} + +variable "cluster_project_id" { + type = string + description = "The GCP project ID where the cluster is created." +} + +variable "regions" { + description = "List of regions where GKE clusters should be created" + type = list(string) + default = ["us-central1"] +} + +variable "gke_cluster_names" { + description = "List of GKE cluster configurations containing cluster name and region" + type = list(string) +} + +# Enable hierarchical namespace GCS buckets +variable "hsn_bucket" { + description = "Enable hierarchical namespace GCS buckets" + type = bool + default = false +} + +# Containers to build +variable "agent_image" { + type = string + description = "Agent image for Cloud Run templates" +} + +# Containers to build +variable "workload_image" { + type = string + description = "Map of image name to configuration (source)" +} + +# Sidecar configuration +variable "workload_grpc_endpoint" { + type = string + description = "Endpoint for Workload that the agent will use" +} + +variable "workload_args" { + type = list(string) + description = "Workload image for Cloud Run templates" +} + +variable "dashboard" { + type = string + default = "dashboards/risk-platform-overview.json" + description = "dashboard JSON configuration. Default is 'dashboards/risk-platform-overview.json'." +} + +# +# Optional functionality +# (Review suggested) +# + +# Configurations to create shell scripts for +variable "test_configs" { + type = map(object({ + parallel = number + testfile = string + })) + default = {} + description = "Test configurations (parallel = 0 use autoscaler)" +} + +variable "workload_init_args" { + type = list(list(string)) + default = [] + description = "Workload initialization arguments to run" +} + +variable "pubsub_exactly_once" { + type = bool + default = true + description = "Enable Pub/Sub exactly once subscriptions" +} + + +# +# Naming defaults +# (Only change if conflicting with other modules) +# + +variable "gke_job_request" { + type = string + default = "gke_job_request" + description = "The identifier for the GKE job request. Default is 'gke_job_request'." +} + +variable "gke_job_response" { + type = string + default = "gke_job_response" + description = "The identifier for the GKE job response. Default is 'gke_job_response'." +} + +variable "gke_hpa_request" { + type = string + default = "gke_hpa_request" + description = "The identifier for the GKE HPA request. Default is 'gke_hpa_request'." +} + +variable "gke_hpa_response" { + type = string + default = "gke_hpa_response" + description = "The identifier for the GKE HPA response. Default is 'gke_hpa_response'." +} + +# Parallelstore +# Enable/disable Parallelstore deployment (default: false) +variable "parallelstore_enabled" { + type = bool + description = "Enable or disable the deployment of Parallelstore." + default = false +} + +variable "parallelstore_instances" { + type = map(object({ + name = string + access_points = list(string) + location = string + region = string + id = string + capacity_gib = number + })) + default = null + description = "Map of parallel store instances configurations. Each instance must have non-null access_points." + validation { + condition = var.parallelstore_instances == null || alltrue([ + for instance in values(var.parallelstore_instances) : + instance.access_points != null && instance.access_points != "" + ]) + error_message = "All parallelstore instances must have non-null access_points" + } +} + +variable "vpc_name" { + type = string + description = "Name of the VPC used by Parallelstore" +} + +variable "env" { + type = string + description = "The environment to prepare (ex. development)" +} + +variable "namespace" { + type = string + description = "The environment's fleet namespace'" +} diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/versions.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/versions.tf new file mode 100644 index 000000000..028dbac5f --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/versions.tf @@ -0,0 +1,35 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/agent/src/.gitignore b/5-appinfra/modules/htc-infra/agent/src/.gitignore new file mode 100644 index 000000000..fd1f11b10 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/.gitignore @@ -0,0 +1,2 @@ +# Produced binary +agent diff --git a/5-appinfra/modules/htc-infra/agent/src/Dockerfile b/5-appinfra/modules/htc-infra/agent/src/Dockerfile new file mode 100644 index 000000000..aedac6760 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.24 AS bld + +# Fetch Dependencies +WORKDIR /build/go +COPY go.* /build/go/ +RUN go mod download + +# Copy in code and build +COPY . /build/go/ +RUN CGO_ENABLED=0 go build + +# Final build +FROM gcr.io/distroless/static-debian12 + +COPY --from=bld /build/go/agent /agent + +# Expose the port 8080 +EXPOSE 8080 + +ENTRYPOINT [ "/agent" ] diff --git a/5-appinfra/modules/htc-infra/agent/src/gcp/google.go b/5-appinfra/modules/htc-infra/agent/src/gcp/google.go new file mode 100644 index 000000000..3a739f57a --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/gcp/google.go @@ -0,0 +1,207 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcp + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + + "cloud.google.com/go/compute/metadata" + "cloud.google.com/go/pubsub" + mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" + "github.com/spf13/cobra" + "go.opencensus.io/stats/view" + "go.opentelemetry.io/contrib/detectors/gcp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/bridge/opencensus" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "golang.org/x/oauth2/google" + "google.golang.org/api/option" +) + +type GoogleConfig struct { + ProjectID string + Region string + Hostname string + + enableOpenTelemetry bool + shutdown func(context.Context) error +} + +func (cfg *GoogleConfig) Initialize(cmd *cobra.Command) { + + credentials, err := google.FindDefaultCredentials(context.Background()) + // Set defaults for project + if envProjectId := os.Getenv("PUBSUB_PROJECT_ID"); envProjectId != "" { + cfg.ProjectID = envProjectId + } else if err == nil { + cfg.ProjectID = credentials.ProjectID + } + + cfg.enableOpenTelemetry = true + + // Fetch hostname (or use Metadata instance if not found) + cfg.Hostname, err = os.Hostname() + if err != nil || cfg.Hostname == "localhost" { + cfg.Hostname, err = metadata.InstanceIDWithContext(context.Background()) + } + if err != nil || cfg.Hostname == "" { + cfg.Hostname = "localhost" + } + + // Set parameters + cmd.PersistentFlags().StringVar(&cfg.ProjectID, "projectId", cfg.ProjectID, "Google Project ID") + cmd.PersistentFlags().StringVar(&cfg.Region, "region", cfg.Region, "Region for regional endpoint(s) (global otherwise)") + cmd.PersistentFlags().BoolVar(&cfg.enableOpenTelemetry, "monitoring", cfg.enableOpenTelemetry, "Enable Cloud Monitoring") +} + +func (cfg *GoogleConfig) getResources(ctxt context.Context) []attribute.KeyValue { + + // Attributes returned + attrs := make([]attribute.KeyValue, 4) + + // If running on Cloud Run.. + kRevision := os.Getenv("K_REVISION") + jobExecution := os.Getenv("CLOUD_RUN_EXECUTION") + if kRevision != "" || jobExecution != "" { + slog.Info("Running on Cloud Run") + + // Namespace (service revision or the job execution) + if kRevision != "" { + attrs = append(attrs, attribute.String("namespace", kRevision)) + } else { + attrs = append(attrs, attribute.String("namespace", jobExecution)) + } + + return attrs + } + + // Open Telemetry capture K8S namespace (if provided) + ns := os.Getenv("K8S_NAMESPACE") + if ns != "" { + slog.Info("Running on Kubernetes") + + attrs = append(attrs, semconv.K8SNamespaceNameKey.String(ns)) + attrs = append(attrs, semconv.K8SPodNameKey.String(cfg.Hostname)) + + return attrs + } + + slog.Info("Uncertain where running - Open Telemetry may have errors") + + return attrs +} + +func (cfg *GoogleConfig) Start(ctxt context.Context, serviceName string) error { + + // Only start once + if cfg.shutdown != nil { + return fmt.Errorf("cannot start Google monitoring twice") + } + + var shutdownFuncs []func(context.Context) error + + // Create shutdown function + cfg.shutdown = func(ctxt context.Context) error { + var err error + for _, fn := range shutdownFuncs { + err = errors.Join(err, fn(ctxt)) + } + shutdownFuncs = nil + return err + } + + // Create exporter + opts := []mexporter.Option{ + mexporter.WithProjectID(cfg.ProjectID), + } + exp, err := mexporter.New(opts...) + if err != nil { + return fmt.Errorf("failed to create exporter: %v", err) + } + + // Identify the resource + res, err := resource.New( + ctxt, + // Use the GCP resource detector to detect information about the GCP platform + resource.WithDetectors(gcp.NewDetector()), + // Keep the default detectors + resource.WithTelemetrySDK(), + // Add attributes from environment variables + resource.WithFromEnv(), + // Add your own custom attributes to identify your application + resource.WithAttributes( + append(cfg.getResources(ctxt), semconv.ServiceNameKey.String(serviceName))..., + ), + ) + if errors.Is(err, resource.ErrPartialResource) || errors.Is(err, resource.ErrSchemaURLConflict) { + slog.Info("Open Telemetry resources partially failed") + } else if err != nil { + return fmt.Errorf("open Telemetry resources failed: %w", err) + } + slog.Info("Open Telemetry resources", "resources", res) + + // Create a bridge from the existing service and load into Open Telemetry + bridge := opencensus.NewMetricProducer() + provider := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exp, sdkmetric.WithProducer(bridge))), + sdkmetric.WithResource(res), + ) + shutdownFuncs = append(shutdownFuncs, provider.Shutdown) + + // Register views + if err := view.Register(pubsub.DefaultSubscribeViews...); err != nil { + slog.Warn("Failed to register view", "error", err) + } + + return nil +} + +func (cfg *GoogleConfig) Stop(ctxt context.Context) error { + if cfg.shutdown != nil { + return cfg.shutdown(ctxt) + } + return nil +} + +func (cfg *GoogleConfig) PubSubClient(ctxt context.Context) (*pubsub.Client, error) { + + // Client Options + opts := []option.ClientOption{} + endpoint := "pubsub.googleapis.com:443" + if cfg.Region != "" { + slog.Info("Connecting to PubSub", "project", cfg.ProjectID, "region", cfg.Region) + endpoint = fmt.Sprintf("%s-pubsub.googleapis.com:443", cfg.Region) + } else { + slog.Info("Connecting to PubSub", "project", cfg.ProjectID) + } + opts = append(opts, option.WithEndpoint(endpoint)) + + // Tracking Options + opts = append(opts, option.WithUserAgent("cloud-solutions/fsi-rdp-agent-v1.0.0")) + + // Create the client + client, err := pubsub.NewClient(ctxt, cfg.ProjectID, opts...) + if err != nil { + return nil, fmt.Errorf("failed creating the client: %w", err) + } + + return client, nil +} diff --git a/5-appinfra/modules/htc-infra/agent/src/gcp/storage.go b/5-appinfra/modules/htc-infra/agent/src/gcp/storage.go new file mode 100644 index 000000000..73e15f67b --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/gcp/storage.go @@ -0,0 +1,155 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcp + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "regexp" + "strings" + + "cloud.google.com/go/storage" + "google.golang.org/api/option" +) + +func (cfg *GoogleConfig) NewStorageClient(ctxt context.Context) (*storage.Client, error) { + return storage.NewClient(ctxt, option.WithUserAgent( + "cloud-solutions/fsi-rdp-agent-v1.0.0")) +} + +type gzipReadWrapper struct { + greader *gzip.Reader + reader io.ReadCloser +} + +func (r gzipReadWrapper) Read(p []byte) (int, error) { + return r.greader.Read(p) +} + +func (r gzipReadWrapper) Close() error { + errGzip := r.greader.Close() + errReader := r.reader.Close() + if errGzip != nil { + if errReader != nil { + slog.Warn("gzipReadWrapper: underlying reader also failed to close", "underlyingError", errReader.Error(), "primaryError", errGzip.Error()) + } + return errGzip + } + return errReader +} + +var gsPattern = regexp.MustCompile(`^gs://([^/]+)/(.*)$`) + +func (cfg *GoogleConfig) OpenReader(ctxt context.Context, file string) (io.ReadCloser, error) { + + // ReadCloser + var r io.ReadCloser + + // if starts with gs://, then open an object instead. + gs_match := gsPattern.FindStringSubmatch(file) + if gs_match != nil { + client, err := cfg.NewStorageClient(ctxt) + if err != nil { + return nil, fmt.Errorf("opening creating GCS client: %w", err) + } + f, err := client.Bucket(gs_match[1]).Object(gs_match[2]).NewReader(ctxt) + if err != nil { + return nil, fmt.Errorf("reading GCS object: %w", err) + } + r = f + } else { + // Open normal file + f, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("opening file %s: %w", file, err) + } + r = f + } + + // If it ends in .gz, filter with gzip + if strings.HasSuffix(file, ".gz") { + greader, err := gzip.NewReader(io.ReadCloser(r)) + if err != nil { + return nil, err + } + r = gzipReadWrapper{ + greader: greader, + reader: r, + } + } + + return r, nil +} + +type gzipWriteWrapper struct { + gwriter *gzip.Writer + writer io.WriteCloser +} + +func (r gzipWriteWrapper) Write(p []byte) (int, error) { + return r.gwriter.Write(p) +} + +func (r gzipWriteWrapper) Close() error { + errGzip := r.gwriter.Close() + errWriter := r.writer.Close() + if errGzip != nil { + if errWriter != nil { + slog.Warn("gzipWriteWrapper: underlying writer also failed to close", "underlyingError", errWriter.Error(), "primaryError", errGzip.Error()) + } + return errGzip + } + return errWriter +} + +func (cfg *GoogleConfig) CreateWriter(ctxt context.Context, file string) (io.WriteCloser, error) { + + var w io.WriteCloser + + // if starts with gs://, then open an object instead. + gs_match := gsPattern.FindStringSubmatch(file) + if gs_match != nil { + client, err := cfg.NewStorageClient(ctxt) + if err != nil { + return nil, fmt.Errorf("opening creating GCS client: %w", err) + } + slog.Debug("Opening GCS for writing", "bucket", gs_match[1], "object", gs_match[2]) + o := client.Bucket(gs_match[1]).Object(gs_match[2]).NewWriter(ctxt) + w = o + } else { + err := os.MkdirAll(filepath.Dir(file), 0750) + if err != nil { + return nil, fmt.Errorf("error creating directory %s: %w", filepath.Dir(file), err) + } + + slog.Debug("Opening file for writing", "file", file) + o, err := os.Create(file) + if err != nil { + return nil, fmt.Errorf("error opening file %s for writing: %w", file, err) + } + w = o + } + + if strings.HasSuffix(file, ".gz") { + w = gzipWriteWrapper{gwriter: gzip.NewWriter(w), writer: w} + } + + return w, nil +} diff --git a/5-appinfra/modules/htc-infra/agent/src/go.mod b/5-appinfra/modules/htc-infra/agent/src/go.mod new file mode 100644 index 000000000..3b329820b --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/go.mod @@ -0,0 +1,67 @@ +module github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent + +go 1.24 +toolchain go1.24.1 + +require ( + cloud.google.com/go/compute/metadata v0.5.2 + cloud.google.com/go/pubsub v1.44.0 + cloud.google.com/go/storage v1.44.0 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.2 + github.com/bufbuild/protocompile v0.14.1 + github.com/jhump/protoreflect v1.17.0 + github.com/spf13/cobra v1.8.1 + go.opencensus.io v0.24.0 + go.opentelemetry.io/contrib/detectors/gcp v1.30.0 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/bridge/opencensus v1.31.0 + go.opentelemetry.io/otel/sdk v1.31.0 + go.opentelemetry.io/otel/sdk/metric v1.31.0 + golang.org/x/oauth2 v0.23.0 + google.golang.org/api v0.203.0 + google.golang.org/grpc v1.67.1 + google.golang.org/protobuf v1.35.1 +) + +require ( + cel.dev/expr v0.16.2 // indirect + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.9.9 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/iam v1.2.1 // indirect + cloud.google.com/go/monitoring v1.21.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect + github.com/envoyproxy/go-control-plane v0.13.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect + golang.org/x/crypto v0.35.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/grpc/stats/opentelemetry v0.0.0-20241010100425-b850ea533f74 // indirect +) diff --git a/5-appinfra/modules/htc-infra/agent/src/go.sum b/5-appinfra/modules/htc-infra/agent/src/go.sum new file mode 100644 index 000000000..e0db06f0c --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/go.sum @@ -0,0 +1,226 @@ +cel.dev/expr v0.16.2 h1:RwRhoH17VhAu9U5CMvMhH1PDVgf0tuz9FT+24AfMLfU= +cel.dev/expr v0.16.2/go.mod h1:gXngZQMkWJoSbE8mOzehJlXQyubn/Vg0vR9/F3W7iw8= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ= +cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= +cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= +cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= +cloud.google.com/go/kms v1.20.0 h1:uKUvjGqbBlI96xGE669hcVnEMw1Px/Mvfa62dhM5UrY= +cloud.google.com/go/kms v1.20.0/go.mod h1:/dMbFF1tLLFnQV44AoI2GlotbjowyUfgVwezxW291fM= +cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= +cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= +cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= +cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= +cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU= +cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= +cloud.google.com/go/pubsub v1.44.0 h1:pLaMJVDTlnUDIKT5L0k53YyLszfBbGoUBo/IqDK/fEI= +cloud.google.com/go/pubsub v1.44.0/go.mod h1:BD4a/kmE8OePyHoa1qAHEw1rMzXX+Pc8Se54T/8mc3I= +cloud.google.com/go/storage v1.44.0 h1:abBzXf4UJKMmQ04xxJf9dYM/fNl24KHoTuBjyJDX2AI= +cloud.google.com/go/storage v1.44.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE= +cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= +cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.2 h1:cZpsGsWTIFKymTA0je7IIvi1O7Es7apb9CF3EQlOcfE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.2/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.2 h1:RopCq1mZTydpZpWfeYDvsnKR5L8VeaNt5JR5wiMfh7Q= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.2/go.mod h1:tlLrnqq33OLuNnYbqswyI5ckZ0QjuM2DFIuaraxxDEU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0 h1:jJKWl98inONJAr/IZrdFQUWcwUO95DLY1XMD1ZIut+g= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0/go.mod h1:l2fIqmwB+FKSfvn3bAD/0i+AXAxhIZjTK2svT/mgUXs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 h1:GYUJLfvd++4DMuMhCFLgLXvFwofIxh/qOwoGuS/LTew= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0/go.mod h1:wRbFgBQUVm1YXrvWKofAEmq9HNJTDphbAaJSSX01KUI= +github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= +github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= +github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= +go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/detectors/gcp v1.30.0 h1:GF+YVnUeJwOy+Ag2cTEpVZq+r2Tnci42FIiNwA2gjME= +go.opentelemetry.io/contrib/detectors/gcp v1.30.0/go.mod h1:p5Av42vWKPezk67MQwLYZwlo/z6xLnN/upaIyQNWBGg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 h1:hCq2hNMwsegUvPzI7sPOvtO9cqyy5GbWt/Ybp2xrx8Q= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0/go.mod h1:LqaApwGx/oUmzsbqxkzuBvyoPpkxk3JQWnqfVrJ3wCA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/bridge/opencensus v1.31.0 h1:YrCZ8NpdMTunNIzRnNoG3KjSLu0PNmRtgtQVJuCxkAQ= +go.opentelemetry.io/otel/bridge/opencensus v1.31.0/go.mod h1:2yEkg7WRb15imAr0jfS4XDNd8LNe/hRES+kFezyO6LI= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU= +google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE= +google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/grpc/stats/opentelemetry v0.0.0-20241010100425-b850ea533f74 h1:lFhLQZFvMWdTzAl3jLgXNBlpf8YUhEEIXnTTZppCFGQ= +google.golang.org/grpc/stats/opentelemetry v0.0.0-20241010100425-b850ea533f74/go.mod h1:xwT0YrcBcgR1ZSSLJtUgCjF5QlvTOhiwA/I9TcYf3Gg= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/5-appinfra/modules/htc-infra/agent/src/main.go b/5-appinfra/modules/htc-infra/agent/src/main.go new file mode 100644 index 000000000..138a792d2 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/main.go @@ -0,0 +1,128 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/serve" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/test" + "github.com/spf13/cobra" +) + +// Description of CLI tool +const LONG_DESCRIPTION = ` +A comprehensive example of many of the ways compute can be orchestrated +on Google Cloud Platform. +` + +func main() { + + // Setup context and cancellation + ctxt, cancel := context.WithCancel(context.Background()) + + // Handle ctrl-C / interrupt + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + cleanup_seconds := 15 + go func() { + + // For each time ctrl-C is run + for range c { + + // If already shutting down, stop immediately + if ctxt.Err() != nil { + slog.Warn("Forcibly exiting on second interrupt during shutdown.") + os.Exit(1) + } + + // Shutdown gracefully + slog.Info("Shutting down") + + // Cancel the main context + cancel() + + // Start monitoring for a forceful shutdown in the background + go func() { + slog.Info(fmt.Sprintf("Waiting %d seconds before exiting forcibly.", cleanup_seconds)) + time.Sleep(time.Second * time.Duration(cleanup_seconds)) + slog.Warn(fmt.Sprintf("Waited %d seconds. Forcibly exiting.", cleanup_seconds)) + os.Exit(1) + }() + } + }() + + // + // Main CLI + // + var statsCfg stats.StatsConfig + var google gcp.GoogleConfig + rootCmd := &cobra.Command{ + Use: "agent", + Short: "agent is an example of running compute on GCP", + Long: LONG_DESCRIPTION, + SilenceUsage: true, + } + + // Capture statistics and logging flags + statsCfg.Initialize(rootCmd, cancel) + + // Capture Google interaction flags + google.Initialize(rootCmd) + + // Add test, serving, and default help commands and flags + rootCmd.AddCommand(serve.AddServeCommands(&statsCfg, &google)) + rootCmd.AddCommand(test.AddTestCommands(&statsCfg, &google)) + + // Initialize default Cobra flags + rootCmd.InitDefaultHelpCmd() + rootCmd.InitDefaultHelpFlag() + rootCmd.CompletionOptions.HiddenDefaultCmd = true + + // Capture any extra arguments from a special environment variable + args := os.Args[1:] + keyValue, ok := os.LookupEnv("HTCAGENT_EXTRA_ARGS") + if ok { + args = append(args, strings.Split(keyValue, ",")...) + } + + // Execute and shutdown quickly on error + rootCmd.SetArgs(args) + if err := rootCmd.ExecuteContext(ctxt); err != nil { + slog.Error("error running", "error", err) + os.Exit(1) + } + + // Shutdown stats and Google (Open Telemetry) + statsCfg.Stop() + _ = google.Stop(context.Background()) + + // See if there was a shutdown due to error + if context.Cause(ctxt) != nil && context.Cause(ctxt) != context.Canceled { + slog.Error("Shutdown due to error", "error", context.Cause(ctxt)) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/backend.go b/5-appinfra/modules/htc-infra/agent/src/protoio/backend.go new file mode 100644 index 000000000..734d01d5f --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/backend.go @@ -0,0 +1,296 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protoio + +import ( + "fmt" + "iter" + "regexp" + "time" + + "context" + + "log/slog" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/jhump/protoreflect/grpcreflect" + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/oauth" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/dynamicpb" +) + +var target_pat = regexp.MustCompilePOSIX(`^(https?)://([^/]+)(:[0-9]+)?/([^/]+)/([^/]+)$`) + +type BackendConfiguration struct { + + // Command line parameters + Endpoint string + Timeout time.Duration + + // Private members + service string + method string + client *grpc.ClientConn + google *gcp.GoogleConfig +} + +func (cfg *BackendConfiguration) Initialize(cmd *cobra.Command, google *gcp.GoogleConfig) { + cfg.google = google + + cmd.PersistentFlags().StringVar(&cfg.Endpoint, "endpoint", cfg.Endpoint, "Target endpoint for backend") + cmd.PersistentFlags().DurationVar(&cfg.Timeout, "timeout", cfg.Timeout, "Connection timeout") +} + +func (cfg *BackendConfiguration) connect(ctxt context.Context) error { + if cfg.client != nil { + return nil + } + + if cfg.Endpoint == "" { + return fmt.Errorf("the target endpoint must be configured") + } + + mat := target_pat.FindStringSubmatch(cfg.Endpoint) + if mat == nil { + return fmt.Errorf("invalid url '%s', expecting https?://(:)?/serviceName/method", cfg.Endpoint) + } + + var hostport string + if len(mat) == 6 { + hostport = mat[2] + mat[3] + cfg.service = mat[4] + cfg.method = mat[5] + } else { + if mat[1] == "http" { + hostport = mat[2] + ":80" + cfg.service = mat[3] + cfg.method = mat[4] + } else { + hostport = mat[2] + ":443" + cfg.service = mat[3] + cfg.method = mat[4] + } + } + + // Configure connection + var opts []grpc.DialOption + if mat[1] == "http" { + slog.Info("Connecting to gRPC server insecure", "hostport", hostport) + opts = []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + } + } else { + slog.Info("Connecting to gRPC server with GCE ADC", "hostport", hostport) + opts = []grpc.DialOption{ + grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), + grpc.WithPerRPCCredentials(oauth.NewComputeEngine()), + } + } + + client, err := grpc.NewClient(hostport, opts...) + if err != nil { + return err + } + + // Start connection + client.Connect() + + // Wait until connected + if cfg.Timeout > 0 { + var currState connectivity.State + connContext, cancelContext := context.WithTimeout(ctxt, cfg.Timeout) + defer cancelContext() + for { + currState = client.GetState() + if currState == connectivity.Ready { + slog.Info("gRPC connected, ready to go.") + break + } + + slog.Info("gRPC not connected, still trying.", "state", currState.String()) + + // Wait for next change + if !client.WaitForStateChange(connContext, currState) { + return fmt.Errorf("failed to connect in timeout of %v", cfg.Timeout) + } + } + } + + // Monitor state for logging + go func() { + var lstate = client.GetState() + + for { + + // Wait for next change + if !client.WaitForStateChange(ctxt, lstate) { + return + } + + lstate = client.GetState() + + slog.Info("gRPC Connection State", "state", lstate.String()) + } + }() + + cfg.client = client + + return nil +} + +func (cfg *BackendConfiguration) GetProtoInvoker(ctxt context.Context, stats *stats.StatsConfig) (func(context.Context, proto.Message) (proto.Message, error), error) { + err := cfg.connect(ctxt) + if err != nil { + return nil, err + } + + if cfg.method == "" { + return nil, fmt.Errorf("the target method must be configured") + } + + _, outputType, err := cfg.GetTypes(ctxt) + if err != nil { + return nil, err + } + + cOpts := []grpc.CallOption{grpc.StaticMethod()} + fullMethod := "/" + cfg.service + "/" + cfg.method + return func(context context.Context, req proto.Message) (proto.Message, error) { + + srcId, ok := ctxt.Value("srcId").(string) + if !ok { + srcId = "" + } + + outmsg := dynamicpb.NewMessage(outputType) + slog.Debug("Invoking", "srcId", srcId, "fullmethod", fullMethod, "request", req) + + stats.StartTask(srcId) + err := cfg.client.Invoke(context, fullMethod, req, outmsg, cOpts...) + stats.DoneTask(srcId, uint64(proto.Size(req))+uint64(proto.Size(outmsg))) + + if err != nil { + return nil, err + } + + return outmsg, nil + }, nil +} + +func (cfg *BackendConfiguration) ReadProtoInput(ctxt context.Context, input string) (iter.Seq2[proto.Message, error], error) { + inputType, _, err := cfg.GetTypes(ctxt) + if err != nil { + return nil, err + } + return ReadProto(ctxt, cfg.google, inputType, input), nil +} + +func (cfg *BackendConfiguration) WriteProtoOutput(ctxt context.Context, iter iter.Seq2[proto.Message, error], output string) error { + return WriteProto(ctxt, cfg.google, iter, output) +} + +func (cfg *BackendConfiguration) GetTypes(ctxt context.Context) (protoreflect.MessageDescriptor, protoreflect.MessageDescriptor, error) { + err := cfg.connect(ctxt) + if err != nil { + return nil, nil, err + } + + if cfg.service == "" { + return nil, nil, fmt.Errorf("the target service must be configured") + } + + reflectclient := grpcreflect.NewClientAuto(ctxt, cfg.client) + desc, err := reflectclient.ResolveService(cfg.service) + if err != nil { + return nil, nil, err + } + + if cfg.method == "" { + return nil, nil, fmt.Errorf("the target method must be configured") + } + + mtd := desc.FindMethodByName(cfg.method) + if mtd == nil { + return nil, nil, fmt.Errorf("service %s does not have a method %s", cfg.service, cfg.method) + } + + /* Validate type of method */ + if mtd.IsClientStreaming() || mtd.IsServerStreaming() { + return nil, nil, fmt.Errorf("streaming is not supported for method %s", cfg.method) + } + + return mtd.GetInputType().UnwrapMessage(), mtd.GetOutputType().UnwrapMessage(), nil +} + +func (cfg *BackendConfiguration) GetJSONConverters(ctxt context.Context) (func([]byte) (proto.Message, error), func(proto.Message) ([]byte, error), error) { + inputType, _, err := cfg.GetTypes(ctxt) + if err != nil { + return nil, nil, err + } + + return JSONToProto(inputType), ProtoToJSON(), nil +} + +func (cfg *BackendConfiguration) GetInvoker(ctxt context.Context, stats *stats.StatsConfig, inJson bool, outJson bool) (func(context.Context, []byte) ([]byte, error), error) { + protoInvoker, err := cfg.GetProtoInvoker(ctxt, stats) + if err != nil { + return nil, err + } + + inputType, _, err := cfg.GetTypes(ctxt) + if err != nil { + return nil, err + } + + var fromBytes func([]byte) (proto.Message, error) + if inJson { + fromBytes = JSONToProto(inputType) + } else { + fromBytes = ProtoBytesToProto(inputType) + } + + var toBytes func(proto.Message) ([]byte, error) + if outJson { + toBytes = ProtoToJSON() + } else { + toBytes = ProtoToProtoBytes() + } + + return func(context context.Context, req []byte) ([]byte, error) { + protoMsg, err := fromBytes(req) + if err != nil { + return nil, err + } + + outMsg, err := protoInvoker(context, protoMsg) + if err != nil { + return nil, err + } + + jsonMsg, err := toBytes(outMsg) + if err != nil { + return nil, err + } + + return jsonMsg, nil + }, nil +} diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/convert.go b/5-appinfra/modules/htc-infra/agent/src/protoio/convert.go new file mode 100644 index 000000000..3297c4925 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/convert.go @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protoio + +import ( + "context" + "fmt" + + "github.com/bufbuild/protocompile" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/dynamicpb" +) + +func CompileDescriptor(ctxt context.Context, protodefinition string) (protoreflect.MessageDescriptor, error) { + + compiler := protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + Accessor: protocompile.SourceAccessorFromMap(map[string]string{ + "main.proto": protodefinition, + }), + }), + MaxParallelism: 1, + SourceInfoMode: protocompile.SourceInfoNone, + } + descs, err := compiler.Compile(ctxt, "main.proto") + if err != nil { + return nil, err + } + if len(descs) != 1 { + return nil, fmt.Errorf("expected to parse one file, got %d files", len(descs)) + } + msgTypes := descs[0].Messages() + if msgTypes.Len() == 0 { + return nil, fmt.Errorf("expected at least one message type defined") + } + + return msgTypes.Get(msgTypes.Len() - 1), nil +} + +func JSONToProto(desc protoreflect.MessageDescriptor) func([]byte) (proto.Message, error) { + opts := &protojson.UnmarshalOptions{ + AllowPartial: false, + DiscardUnknown: true, + } + + return func(jsonmsg []byte) (proto.Message, error) { + if desc == nil { + return nil, fmt.Errorf("missing descriptor") + } + msg := dynamicpb.NewMessage(desc) + if err := opts.Unmarshal([]byte(jsonmsg), msg); err != nil { + return nil, fmt.Errorf("error marshing JSON to protobuf: %w", err) + } + return msg, nil + } +} + +func ProtoBytesToProto(desc protoreflect.MessageDescriptor) func([]byte) (proto.Message, error) { + return func(req []byte) (proto.Message, error) { + msg := dynamicpb.NewMessage(desc) + if err := proto.Unmarshal(req, msg); err != nil { + return nil, fmt.Errorf("error marshing bytes to JSON: %w", err) + } + return msg, nil + } +} + +func JSONToProtoBytes(desc protoreflect.MessageDescriptor) func([]byte) ([]byte, error) { + jsonToProto := JSONToProto(desc) + + return func(line []byte) ([]byte, error) { + msg, err := jsonToProto(line) + if err != nil { + return nil, err + } + msgdata, err := proto.Marshal(msg) + if err != nil { + return nil, err + } + return msgdata, nil + } +} + +func ProtoToProtoBytes() func(proto.Message) ([]byte, error) { + return proto.Marshal +} + +func ProtoToJSON() func(proto.Message) ([]byte, error) { + opts := &protojson.MarshalOptions{ + Multiline: false, + Indent: "", + AllowPartial: false, + EmitDefaultValues: false, + EmitUnpopulated: false, + UseEnumNumbers: true, + UseProtoNames: true, + } + + return func(msg proto.Message) ([]byte, error) { + return opts.Marshal(msg) + } +} diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/io.go b/5-appinfra/modules/htc-infra/agent/src/protoio/io.go new file mode 100644 index 000000000..d0638190d --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/io.go @@ -0,0 +1,144 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protoio + +import ( + "bufio" + "context" + "fmt" + "io" + "iter" + "log/slog" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +// ReadLines (text) +func ReadLines(ctxt context.Context, google *gcp.GoogleConfig, input string) iter.Seq2[[]byte, error] { + return func(yield func(msg []byte, err error) bool) { + + r, err := google.OpenReader(ctxt, input) + if err != nil { + yield(nil, err) + return + } + defer func() { + if closeErr := r.Close(); closeErr != nil { + slog.Error("Failed to close reader in ReadLines iterator", "input", input, "error", closeErr.Error()) + } + }() + + reader := bufio.NewReader(r) + + for ctxt.Err() == nil { + msg, err := reader.ReadString('\n') + if err != nil { + if err != io.EOF { + yield(nil, err) + } + return + } + if !yield([]byte(msg), nil) { + return + } + } + } +} + +// WriteLines (text) +func WriteLines(ctxt context.Context, google *gcp.GoogleConfig, src iter.Seq2[[]byte, error], output string) (retErr error) { + w, err := google.CreateWriter(ctxt, output) + if err != nil { + retErr = fmt.Errorf("failed to create writer for '%s': %w", output, err) + return + } + + defer func() { + if closeErr := w.Close(); closeErr != nil { + if retErr == nil { + retErr = fmt.Errorf("failed to close writer for '%s': %w", output, closeErr) + } else { + slog.Error("Additionally failed to close writer after a primary write error", + "output", output, + "primaryError", retErr.Error(), + "closeError", closeErr.Error()) + } + } + }() + + for msg, iterErr := range src { + if ctxt.Err() != nil { + retErr = ctxt.Err() + return + } + if iterErr != nil { + retErr = fmt.Errorf("error from source iterator while writing to '%s': %w", output, iterErr) + return + } + _, writeErr := w.Write(msg) + if writeErr != nil { + retErr = fmt.Errorf("failed to write to '%s': %w", output, writeErr) + return + } + _, writeErr = w.Write([]byte{'\n'}) + if writeErr != nil { + retErr = fmt.Errorf("failed to write to '%s': %w", output, writeErr) + return + } + } + + if ctxt.Err() != nil { + retErr = ctxt.Err() + } + return +} + +// Read as Proto messages +func ReadProto(ctxt context.Context, google *gcp.GoogleConfig, desc protoreflect.MessageDescriptor, input string) iter.Seq2[proto.Message, error] { + return MapIterErr(ReadLines(ctxt, google, input), JSONToProto(desc)) +} + +// Read as Proto messages as bytes +func ReadProtoBytes(ctxt context.Context, google *gcp.GoogleConfig, desc protoreflect.MessageDescriptor, input string) iter.Seq2[[]byte, error] { + return MapIterErr(ReadLines(ctxt, google, input), JSONToProtoBytes(desc)) +} + +// Write as Proto messages +func WriteProto(ctxt context.Context, google *gcp.GoogleConfig, src iter.Seq2[proto.Message, error], output string) error { + return WriteLines(ctxt, google, MapIterErr(src, ProtoToJSON()), output) +} + +// Map input to output through a filter, aborting on any error +func MapIterErr[I any, O any](input iter.Seq2[I, error], filterFunc func(I) (O, error)) iter.Seq2[O, error] { + return func(yield func(o O, err error) bool) { + var empty O + for i, err := range input { + if err != nil { + yield(empty, err) + return + } + o, err := filterFunc(i) + if err != nil { + yield(empty, err) + return + } + if !yield(o, nil) { + return + } + } + } +} diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go b/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go new file mode 100644 index 000000000..c52e9529b --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go @@ -0,0 +1,122 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protoio + +import ( + "context" + "path" + "testing" + + "google.golang.org/protobuf/proto" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" +) + +// Command to compile the protobuf for tests: +// protoc -I. --go-grpc_out=. --go-grpc_opt=paths=source_relative test_service.proto +// protoc -I. --go_out=. --go_opt=paths=source_relative test_service.proto + +var defn = ` +syntax = "proto3"; + +package main; + +message QuantTask { + + // Unique identifier (possibly a timestamp!) + int64 id = 1; + + // Likelihood of failure (for a simulator) + double perc_crash = 2; + double perc_fail = 3; + + // How long it takes to run (for a simulator) + int64 max_micros = 4; + int64 min_micros = 5; + + // Size of payload results (all risk metrics, sensitivities, etc) + int64 result_size = 6; + + // Payload includes all marketdata, trade information, + // risk metric parameters, etc.. + bytes payload = 7; +} +` + +var json_msg = ` +{ "id": 1, "perc_crash":0.1, "perc_fail":0.3, "max_micros": 10000, "min_micros": 1000, "result_size":10, "payload": "alskdfjlkj3214"} +` + +func TestProtoDefinition(t *testing.T) { + desc, err := CompileDescriptor(context.Background(), defn) + if err != nil { + t.Fatalf("Failed compilng %s: %v", defn, err) + } + + fromJsonConverter := JSONToProto(desc) + fromProtoConverter := ProtoToJSON() + + asProto, err := fromJsonConverter([]byte(json_msg)) + if err != nil { + t.Fatalf("failed converting from json %s to proto: %v", json_msg, err) + } + + asJson, err := fromProtoConverter(asProto) + if err != nil { + t.Fatalf("failed converting proto to json: %v", err) + } + t.Logf("Original vs converted json: %s vs %s", json_msg, asJson) + + //ReadProto(context.Background(), desc, "testInput") + // tmpdir := t.TempDir() + tmpdir := "/tmp" + tmpFile := path.Join(tmpdir, "testOutput") + + // Produce ten in a row! + inputSrc := func(yield func(proto.Message, error) bool) { + for range 10 { + if !yield(asProto, nil) { + break + } + } + } + + // This is only relevant when reading/writing GCS, but needs to be supplied + google := &gcp.GoogleConfig{} + + err = WriteProto(context.Background(), google, inputSrc, tmpFile+".jsonl") + if err != nil { + t.Fatalf("Failed writing to avro: %v", err) + } + + seq := ReadProto(context.Background(), google, desc, tmpFile+".jsonl") + + err = WriteProto(context.Background(), google, seq, tmpFile+".jsonl.gz") + if err != nil { + t.Fatalf("Failed writing jsonl: %v", err) + } + + seq2 := ReadProto(context.Background(), google, desc, tmpFile+".jsonl.gz") + if err != nil { + t.Fatalf("Failed reading from avro: %v", err) + } + + for msg, err := range seq2 { + if err != nil { + t.Fatalf("Failed reading from avro: %v", err) + } + t.Logf("Read message: %v", msg) + } +} diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.pb.go b/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.pb.go new file mode 100644 index 000000000..6512e0201 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.pb.go @@ -0,0 +1,271 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.3 +// protoc v3.21.12 +// source: test_service.proto + +package protoio + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type TestInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Unique identifier (possibly a timestamp!) + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + // Likelihood of failure (for a simulator) + PercCrash float64 `protobuf:"fixed64,2,opt,name=perc_crash,json=percCrash,proto3" json:"perc_crash,omitempty"` + PercFail float64 `protobuf:"fixed64,3,opt,name=perc_fail,json=percFail,proto3" json:"perc_fail,omitempty"` + // How long it takes to run (for a simulator) + MaxMicros int64 `protobuf:"varint,4,opt,name=max_micros,json=maxMicros,proto3" json:"max_micros,omitempty"` + MinMicros int64 `protobuf:"varint,5,opt,name=min_micros,json=minMicros,proto3" json:"min_micros,omitempty"` + // Size of payload results (all risk metrics, sensitivities, etc) + ResultSize int64 `protobuf:"varint,6,opt,name=result_size,json=resultSize,proto3" json:"result_size,omitempty"` + // Payload includes all marketdata, trade information, + // risk metric parameters, etc.. + Payload []byte `protobuf:"bytes,7,opt,name=payload,proto3" json:"payload,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TestInput) Reset() { + *x = TestInput{} + mi := &file_test_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TestInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestInput) ProtoMessage() {} + +func (x *TestInput) ProtoReflect() protoreflect.Message { + mi := &file_test_service_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestInput.ProtoReflect.Descriptor instead. +func (*TestInput) Descriptor() ([]byte, []int) { + return file_test_service_proto_rawDescGZIP(), []int{0} +} + +func (x *TestInput) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *TestInput) GetPercCrash() float64 { + if x != nil { + return x.PercCrash + } + return 0 +} + +func (x *TestInput) GetPercFail() float64 { + if x != nil { + return x.PercFail + } + return 0 +} + +func (x *TestInput) GetMaxMicros() int64 { + if x != nil { + return x.MaxMicros + } + return 0 +} + +func (x *TestInput) GetMinMicros() int64 { + if x != nil { + return x.MinMicros + } + return 0 +} + +func (x *TestInput) GetResultSize() int64 { + if x != nil { + return x.ResultSize + } + return 0 +} + +func (x *TestInput) GetPayload() []byte { + if x != nil { + return x.Payload + } + return nil +} + +type TestOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TestOutput) Reset() { + *x = TestOutput{} + mi := &file_test_service_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TestOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestOutput) ProtoMessage() {} + +func (x *TestOutput) ProtoReflect() protoreflect.Message { + mi := &file_test_service_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestOutput.ProtoReflect.Descriptor instead. +func (*TestOutput) Descriptor() ([]byte, []int) { + return file_test_service_proto_rawDescGZIP(), []int{1} +} + +func (x *TestOutput) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *TestOutput) GetPayload() []byte { + if x != nil { + return x.Payload + } + return nil +} + +var File_test_service_proto protoreflect.FileDescriptor + +var file_test_service_proto_rawDesc = []byte{ + 0x0a, 0x12, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0xd0, 0x01, 0x0a, 0x09, 0x54, + 0x65, 0x73, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x65, 0x72, 0x63, + 0x5f, 0x63, 0x72, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x70, 0x65, + 0x72, 0x63, 0x43, 0x72, 0x61, 0x73, 0x68, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x65, 0x72, 0x63, 0x5f, + 0x66, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x70, 0x65, 0x72, 0x63, + 0x46, 0x61, 0x69, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x61, 0x78, 0x5f, 0x6d, 0x69, 0x63, 0x72, + 0x6f, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x61, 0x78, 0x4d, 0x69, 0x63, + 0x72, 0x6f, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x5f, 0x6d, 0x69, 0x63, 0x72, 0x6f, + 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x69, 0x6e, 0x4d, 0x69, 0x63, 0x72, + 0x6f, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53, + 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x36, 0x0a, + 0x0a, 0x54, 0x65, 0x73, 0x74, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x32, 0x3d, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x2e, 0x0a, 0x07, 0x52, 0x75, 0x6e, 0x54, 0x65, 0x73, 0x74, 0x12, + 0x0f, 0x2e, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, + 0x1a, 0x10, 0x2e, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x4f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x22, 0x00, 0x42, 0x5b, 0x5a, 0x59, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x50, 0x6c, + 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x66, 0x69, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x2d, 0x72, + 0x65, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2d, 0x72, 0x69, 0x73, 0x6b, 0x2d, 0x65, 0x78, 0x61, + 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x72, + 0x69, 0x73, 0x6b, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x69, + 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_test_service_proto_rawDescOnce sync.Once + file_test_service_proto_rawDescData = file_test_service_proto_rawDesc +) + +func file_test_service_proto_rawDescGZIP() []byte { + file_test_service_proto_rawDescOnce.Do(func() { + file_test_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_service_proto_rawDescData) + }) + return file_test_service_proto_rawDescData +} + +var file_test_service_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_test_service_proto_goTypes = []any{ + (*TestInput)(nil), // 0: main.TestInput + (*TestOutput)(nil), // 1: main.TestOutput +} +var file_test_service_proto_depIdxs = []int32{ + 0, // 0: main.TestService.RunTest:input_type -> main.TestInput + 1, // 1: main.TestService.RunTest:output_type -> main.TestOutput + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_test_service_proto_init() } +func file_test_service_proto_init() { + if File_test_service_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_test_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_test_service_proto_goTypes, + DependencyIndexes: file_test_service_proto_depIdxs, + MessageInfos: file_test_service_proto_msgTypes, + }.Build() + File_test_service_proto = out.File + file_test_service_proto_rawDesc = nil + file_test_service_proto_goTypes = nil + file_test_service_proto_depIdxs = nil +} diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.proto b/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.proto new file mode 100644 index 000000000..36657a536 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.proto @@ -0,0 +1,51 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package main; + +option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio"; + +service TestService { + + // Runs the simulated library + rpc RunTest(TestInput) returns (TestOutput) {} +} + +message TestInput { + + // Unique identifier (possibly a timestamp!) + int64 id = 1; + + // Likelihood of failure (for a simulator) + double perc_crash = 2; + double perc_fail = 3; + + // How long it takes to run (for a simulator) + int64 max_micros = 4; + int64 min_micros = 5; + + // Size of payload results (all risk metrics, sensitivities, etc) + int64 result_size = 6; + + // Payload includes all marketdata, trade information, + // risk metric parameters, etc.. + bytes payload = 7; +} + +message TestOutput { + int64 id = 1; + bytes payload = 2; +} diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/test_service_grpc.pb.go b/5-appinfra/modules/htc-infra/agent/src/protoio/test_service_grpc.pb.go new file mode 100644 index 000000000..817376002 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/test_service_grpc.pb.go @@ -0,0 +1,137 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.21.12 +// source: test_service.proto + +package protoio + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + TestService_RunTest_FullMethodName = "/main.TestService/RunTest" +) + +// TestServiceClient is the client API for TestService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type TestServiceClient interface { + // Runs the simulated library + RunTest(ctx context.Context, in *TestInput, opts ...grpc.CallOption) (*TestOutput, error) +} + +type testServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient { + return &testServiceClient{cc} +} + +func (c *testServiceClient) RunTest(ctx context.Context, in *TestInput, opts ...grpc.CallOption) (*TestOutput, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(TestOutput) + err := c.cc.Invoke(ctx, TestService_RunTest_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// TestServiceServer is the server API for TestService service. +// All implementations must embed UnimplementedTestServiceServer +// for forward compatibility. +type TestServiceServer interface { + // Runs the simulated library + RunTest(context.Context, *TestInput) (*TestOutput, error) + mustEmbedUnimplementedTestServiceServer() +} + +// UnimplementedTestServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedTestServiceServer struct{} + +func (UnimplementedTestServiceServer) RunTest(context.Context, *TestInput) (*TestOutput, error) { + return nil, status.Errorf(codes.Unimplemented, "method RunTest not implemented") +} +func (UnimplementedTestServiceServer) mustEmbedUnimplementedTestServiceServer() {} +func (UnimplementedTestServiceServer) testEmbeddedByValue() {} + +// UnsafeTestServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to TestServiceServer will +// result in compilation errors. +type UnsafeTestServiceServer interface { + mustEmbedUnimplementedTestServiceServer() +} + +func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) { + // If the following call pancis, it indicates UnimplementedTestServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&TestService_ServiceDesc, srv) +} + +func _TestService_RunTest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TestInput) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TestServiceServer).RunTest(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TestService_RunTest_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TestServiceServer).RunTest(ctx, req.(*TestInput)) + } + return interceptor(ctx, in, info, handler) +} + +// TestService_ServiceDesc is the grpc.ServiceDesc for TestService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var TestService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "main.TestService", + HandlerType: (*TestServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "RunTest", + Handler: _TestService_RunTest_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "test_service.proto", +} diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/bigquery_rdf.go b/5-appinfra/modules/htc-infra/agent/src/serve/bigquery_rdf.go new file mode 100644 index 000000000..fc0c92c2a --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/serve/bigquery_rdf.go @@ -0,0 +1,149 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package serve + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" +) + +type contextKey string + +const srcIDKey contextKey = "srcId" + +func getRDFCommand(cfg *protoio.BackendConfiguration, stats *stats.StatsConfig) *cobra.Command { + grpcCmd := &cobra.Command{ + Use: "rdf", + Short: "BigQuery RDF Server", + Long: "BigQuery RDF Server (port comes from PORT environment variable or 8080)", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + invoker, err := cfg.GetInvoker(cmd.Context(), stats, true, true) + if err != nil { + return err + } + + return handleBigQuery(cmd.Context(), invoker) + }, + } + + return grpcCmd +} + +type BigQueryRDFRequest struct { + RequestId string `json:"requestId"` + Caller string `json:"caller"` + SessionUser string `json:"sessionUser"` + UserDefinedContext map[string]string `json:"userDefinedContext"` + Calls [][]json.RawMessage `json:"calls"` +} + +type BigQueryRDFResponse struct { + Replies []*json.RawMessage `json:"replies,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` +} + +func handleBigQuery(ctxt context.Context, invoker func(context.Context, []byte) ([]byte, error)) error { + + // Handler for BigQuery RDF + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + + // Read message + body, err := io.ReadAll(r.Body) + if err != nil { + slog.Warn("Error reading HTTP request", "error", err) + http.Error(w, fmt.Sprintf("io.ReadAll: %v", err), http.StatusBadRequest) + return + } + + // Decode message + var m BigQueryRDFRequest + if err := json.Unmarshal(body, &m); err != nil { + slog.Warn("Error json unmarshal", "error", err) + http.Error(w, fmt.Sprintf("json.Unmarshal: %v", err), http.StatusBadRequest) + return + } + + slog.Debug("Reading message", "rdfrequest", m) + + // Call the routine serially + result := &BigQueryRDFResponse{} + result.Replies = make([]*json.RawMessage, len(m.Calls)) + for i, callData := range m.Calls { + slog.Debug("Running on backend as JSON", "json", string(callData[0])) + requestCtx := context.WithValue(ctxt, srcIDKey, m.RequestId) + reply, err := invoker( + requestCtx, + callData[0]) + + if err != nil { + slog.Warn("Error running on backend", "error", err) + result.ErrorMessage = fmt.Sprintf("error calling backend: %v", err) + break + } + + result.Replies[i] = (*json.RawMessage)(&reply) + } + + // Send back the response + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + slog.Debug("Returning message", "json", result) + b, err := json.Marshal(result) + if err != nil { + slog.Warn("Error marshing response", "error", err) + } else { + slog.Debug("Returning", "encoded", b) + } + if err := json.NewEncoder(w).Encode(result); err != nil { + slog.Warn("Error encoding response", "error", err) + } + }) + + port := os.Getenv("PORT") + if port == "" { + port = "8080" + } + + // Start HTTP server. + srv := &http.Server{ + Addr: ":" + port, + } + go func() { + <-ctxt.Done() + slog.Info("Shutting down HTTP Server") + if err := srv.Shutdown(context.Background()); err != nil { + slog.Warn("Error shutting down HTTP Server", "error", err) + } + }() + + slog.Info("Listening on port", "port", port) + err := srv.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + return fmt.Errorf("error listening: %w", err) + } + + return nil +} diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/file.go b/5-appinfra/modules/htc-infra/agent/src/serve/file.go new file mode 100644 index 000000000..1a7b27f0b --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/serve/file.go @@ -0,0 +1,69 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package serve + +import ( + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" + "google.golang.org/protobuf/proto" +) + +func getFileCommand(cfg *protoio.BackendConfiguration, stats *stats.StatsConfig) *cobra.Command { + cmd := &cobra.Command{ + Use: "file input-file output-output", + Short: "Read and write JSONL or gzipped JSONL for work", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + input := args[0] + output := args[1] + + invoker, err := cfg.GetProtoInvoker(cmd.Context(), stats) + if err != nil { + return err + } + + // Read directly (serially) + r, err := cfg.ReadProtoInput(cmd.Context(), input) + if err != nil { + return err + } + + // Get the processed messages + processed := func(yield func(proto.Message, error) bool) { + for inmsg := range r { + outmsg, err := invoker(cmd.Context(), inmsg) + if err != nil { + yield(nil, err) + return + } + + if !yield(outmsg, nil) { + return + } + + } + } + + // Write the output (serially) + if err := cfg.WriteProtoOutput(cmd.Context(), processed, output); err != nil { + return err + } + + return nil + }, + } + return cmd +} diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/pubsub.go b/5-appinfra/modules/htc-infra/agent/src/serve/pubsub.go new file mode 100644 index 000000000..49a5d4c22 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/serve/pubsub.go @@ -0,0 +1,168 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package serve + +import ( + "context" + "fmt" + "log/slog" + "time" + + "cloud.google.com/go/pubsub" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" +) + +func NewPubSubPullAgent(cfg *protoio.BackendConfiguration, stats *stats.StatsConfig, google *gcp.GoogleConfig) *cobra.Command { + var idleTimeout time.Duration + var jsonPubSub bool + + subSettings := &pubsub.ReceiveSettings{} + cmd := &cobra.Command{ + Use: "pubsub-pull subscription-id topic-id", + Short: "PubSub pull and publish worker", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + invoker, err := cfg.GetInvoker(cmd.Context(), stats, jsonPubSub, jsonPubSub) + if err != nil { + return err + } + return handlePubSubSubscribe(cmd.Context(), stats, google, invoker, subSettings, idleTimeout, google.ProjectID, args[0], args[1], jsonPubSub) + }, + } + cmd.Flags().DurationVar(&idleTimeout, "idleTimeout", idleTimeout, + "Idle timeout -- shutdown after no activity for this time period") + cmd.Flags().DurationVar(&subSettings.MinExtensionPeriod, "minExtensionPeriod", 600*time.Second, + "Minimum deadline extension for processing tasks") + cmd.Flags().DurationVar(&subSettings.MaxExtension, "maxExtension", 60*time.Minute, + "Maximum deadline for processing tasks") + cmd.Flags().IntVar(&subSettings.NumGoroutines, "goroutines", 1, + "Goroutine count") + cmd.Flags().IntVar(&subSettings.MaxOutstandingMessages, "maxoutstandingmessages", 1, + "Max number of messages outstanding") + cmd.Flags().BoolVar(&subSettings.Synchronous, "synchronous", false, + "Enable synchronous mode for subscriptions") + cmd.Flags().BoolVar(&jsonPubSub, "jsonPubSub", true, + "Enable JSON in Pub/Sub instead of byte-encoded protobuf") + + return cmd +} + +func handlePubSubSubscribe( + ctxt context.Context, + stats *stats.StatsConfig, + google *gcp.GoogleConfig, + invoker func(context.Context, []byte) ([]byte, error), + settings *pubsub.ReceiveSettings, idleTimeout time.Duration, + project string, subscription string, topic string, jsonPubSub bool) error { + + slog.Debug("Subscribing", "project", project, "subscription", subscription) + + // Create PubSub client + client, err := google.PubSubClient(ctxt) + if err != nil { + return fmt.Errorf("creating client: %w", err) + } + + // Publish the topic response + top := client.TopicInProject(topic, project) + + // Prepare to shutdown after an idle period + if idleTimeout > time.Duration(0) { + slog.Info("Shutting down after an idle timeout", "idleTimeout", idleTimeout) + stats.SetStatsIdleTimeout(idleTimeout) + } + + // Subscribe to data + sub := client.SubscriptionInProject(subscription, project) + sub.ReceiveSettings = *settings + return sub.Receive(ctxt, func(ctxt context.Context, msg *pubsub.Message) { + + slog.Debug("received request", + "data", msg.Data, + "attributes", msg.Attributes) + + res, err := handleMessage(ctxt, google, invoker, top, msg.Data, msg.Attributes) + if err != nil { + slog.Warn("Failed calling service", "error", err) + logAckResult(ctxt, msg.NackWithResult()) + return + } + + go func() { + + // Wait for publish acknowledgement + <-res.Ready() + + // Nack on failure + if _, err := res.Get(ctxt); err != nil { + slog.Warn("failed publishing result", "error", err) + logAckResult(ctxt, msg.NackWithResult()) + return + } + + // All good - ack message! + logAckResult(ctxt, msg.AckWithResult()) + }() + }) +} + +func logAckResult(ctxt context.Context, r *pubsub.AckResult) { + status, err := r.Get(ctxt) + if err != nil { + slog.Warn("Failed when calling result.Get", "error", err) + } + if status != pubsub.AcknowledgeStatusSuccess { + slog.Warn("Message acknowledged failed", "status", status) + } +} + +func handleMessage( + ctxt context.Context, + google *gcp.GoogleConfig, + invoker func(context.Context, []byte) ([]byte, error), + top *pubsub.Topic, + data []byte, + attributes map[string]string) (*pubsub.PublishResult, error) { + + // Extract srcId + srcId, ok := attributes["srcId"] + if !ok { + srcId = "" + } + + // Run the task raw + requestCtx := context.WithValue(ctxt, srcIDKey, srcId) + rbuf, err := invoker( + requestCtx, + data) + + // If error, do a Nack for faster retry + if err != nil { + return nil, fmt.Errorf("error running library: %w", err) + } + + // Response with some attributes for incoming message! + attributes["Hostname"] = google.Hostname + rmsg := &pubsub.Message{ + Data: rbuf, + Attributes: attributes, + } + + // Publish and ack when done + return top.Publish(ctxt, rmsg), nil +} diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/pubsub_push.go b/5-appinfra/modules/htc-infra/agent/src/serve/pubsub_push.go new file mode 100644 index 000000000..b473f1d6b --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/serve/pubsub_push.go @@ -0,0 +1,138 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package serve + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" +) + +func NewPubSubPushAgent(cfg *protoio.BackendConfiguration, stats *stats.StatsConfig, google *gcp.GoogleConfig) *cobra.Command { + + // PubSub push + var jsonPubSub bool + cmd := &cobra.Command{ + Use: "pubsub-push topic-id", + Short: "Cloud Run PubSub endpoint listener and publish worker", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + invoker, err := cfg.GetInvoker(cmd.Context(), stats, true, jsonPubSub) + if err != nil { + return err + } + return handlePubSubPushSubscription(cmd.Context(), invoker, google, args[0]) + }, + } + cmd.Flags().BoolVar( + &jsonPubSub, + "jsonPubSub", + true, + "Enable JSON in Pub/Sub instead of byte-encoded protobuf") + + return cmd +} + +func handlePubSubPushSubscription( + ctxt context.Context, + invoker func(context.Context, []byte) ([]byte, error), + google *gcp.GoogleConfig, + topic string) error { + + // Create client for publishing + client, err := google.PubSubClient(ctxt) + if err != nil { + slog.Warn("error creating client", "error", err) + return err + } + + // Publishing topic + top := client.TopicInProject(topic, google.ProjectID) + + // PubSubMessage is the payload of a Pub/Sub event. + // See the documentation for more details: + // https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage + type PubSubMessage struct { + Message struct { + Data []byte `json:"data,omitempty"` + Attributes map[string]string `json:"attributes"` + ID string `json:"messageId"` + } `json:"message"` + Subscription string `json:"subscription"` + } + + // Handler for the push subscription + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + + // Read message + var m PubSubMessage + body, err := io.ReadAll(r.Body) + if err != nil { + slog.Warn("error reading message", "error", err) + http.Error(w, fmt.Sprintf("io.ReadAll: %v", err), http.StatusBadRequest) + return + } + + // byte slice unmarshalling handles base64 decoding. + if err := json.Unmarshal(body, &m); err != nil { + slog.Warn("error unmarshalling JSON", "error", err, "body", body) + http.Error(w, fmt.Sprintf("json.Unmarshal: %v", err), http.StatusBadRequest) + return + } + + // handleMessage is re-used from the general pubsub code + res, err := handleMessage(ctxt, google, invoker, top, m.Message.Data, m.Message.Attributes) + if err != nil { + slog.Warn("failed handling message", "error", err) + http.Error(w, fmt.Sprintf("failed handling message: %v", err), http.StatusBadRequest) + return + } + + // Wait for publishing to finish + <-res.Ready() + if _, err := res.Get(ctxt); err != nil { + slog.Warn("failed publishing result", "error", err) + http.Error(w, fmt.Sprintf("failed publishing result: %v", err), http.StatusBadRequest) + return + } + + // Ack the message + w.WriteHeader(http.StatusOK) + }) + + port := os.Getenv("PORT") + if port == "" { + port = "8080" + slog.Info("defaulting to port", "port", port) + } + + // Start HTTP server. + slog.Info("Listening", "port", port) + if err := http.ListenAndServe(":"+port, nil); err != nil { + slog.Info("error listening", "error", err) + return err + } + + return nil +} diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/serve.go b/5-appinfra/modules/htc-infra/agent/src/serve/serve.go new file mode 100644 index 000000000..82e20271a --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/serve/serve.go @@ -0,0 +1,47 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package serve + +import ( + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" +) + +func AddServeCommands(stats *stats.StatsConfig, google *gcp.GoogleConfig) *cobra.Command { + + cfg := &protoio.BackendConfiguration{} + cmd := &cobra.Command{ + Use: "serve", + Short: "Serve work a sidecar worker", + Args: cobra.ExactArgs(0), + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + stats.Start(cmd.Context()) + if err := google.Start(cmd.Context(), "agent"); err != nil { + return err + } + return nil + }, + } + cfg.Initialize(cmd, google) + + cmd.AddCommand(NewPubSubPullAgent(cfg, stats, google)) + cmd.AddCommand(NewPubSubPushAgent(cfg, stats, google)) + cmd.AddCommand(getRDFCommand(cfg, stats)) + cmd.AddCommand(getFileCommand(cfg, stats)) + + return cmd +} diff --git a/5-appinfra/modules/htc-infra/agent/src/stats/stats.go b/5-appinfra/modules/htc-infra/agent/src/stats/stats.go new file mode 100644 index 000000000..1951f507c --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/stats/stats.go @@ -0,0 +1,192 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stats + +import ( + "context" + "log/slog" + "os" + "sync" + "time" + + "github.com/spf13/cobra" +) + +type statOperation int + +const ( + idStart statOperation = iota + idCancel + idDone + + idleTimeoutSet +) + +type stats struct { + Operation statOperation + Id string + Bytes uint64 + Timeout time.Duration +} + +// ShowStats in a certain frequency. This logs using slog. +func (cfg *StatsConfig) showStats( + ctxt context.Context) { + + cfg.statsShutdown.Add(1) + go func() { + defer cfg.statsShutdown.Done() + + accum := NewAccumulatedStats() + + timeout := time.Duration(-1) + + // Last idle time (default to zero) + var idleTime time.Time + + // Stat accumulation time + ntime := time.Now().Add(cfg.Freq) + + timeoutTimer := time.NewTimer(timeout) + timeoutTimer.Stop() + + StatsLoop: + for ctxt.Err() == nil { + + if idleTime.IsZero() { + timeoutTimer.Stop() + } else if timeout >= time.Duration(0) { + timerLimit := timeout - time.Since(idleTime) + if timerLimit <= time.Duration(0) { + cfg.cancel() + } else { + timeoutTimer.Reset(timerLimit) + } + } + + select { + case <-ctxt.Done(): + break StatsLoop + + case e, ok := <-cfg.statsChannel: + if !ok { + break StatsLoop + } + switch e.Operation { + case idleTimeoutSet: + timeout = e.Timeout + case idStart: + if cfg.LogAll { + slog.Info("taskStart", + "id", e.Id, + "active", accum.ActiveOp()) + } + accum.StartOp(e.Id) + case idCancel: + accum.CancelOp(e.Id) + case idDone: + runtime := accum.DoneOp(e.Id, e.Bytes) + + if cfg.LogAll { + slog.Info("task", + "id", e.Id, + "runtime", runtime, + "bytes", e.Bytes) + } + } + + // Idle the idle time if needed + if accum.ActiveOp() == 0 && idleTime.IsZero() { + idleTime = time.Now() + } else if accum.ActiveOp() > 0 && !idleTime.IsZero() { + idleTime = time.Time{} + } + + case <-time.After(time.Until(ntime)): + accum.LogStats(true, cfg.LogJSON) + + // Next time + ntime = ntime.Add(cfg.Freq) + + case <-timeoutTimer.C: + cfg.cancel() + } + + } + + // Shutdown final statistics + accum.LogStats(true, cfg.LogJSON) + }() +} + +type StatsConfig struct { + LogAll bool + LogJSON bool + Debug bool + Freq time.Duration + + cancel context.CancelFunc + statsChannel chan stats + statsShutdown sync.WaitGroup +} + +func (cfg *StatsConfig) Initialize(cmd *cobra.Command, cancel context.CancelFunc) { + cfg.Freq = time.Second * 5 + cfg.cancel = cancel + cfg.statsChannel = make(chan stats) + + cmd.PersistentFlags().DurationVar(&cfg.Freq, "freq", 5*time.Second, "Frequency of logging statistics") + cmd.PersistentFlags().BoolVar(&cfg.Debug, "debug", false, "Enable debug logging") + cmd.PersistentFlags().BoolVar(&cfg.LogAll, "logAll", false, "Enable logging each completed operation") + cmd.PersistentFlags().BoolVar(&cfg.LogJSON, "logJSON", false, "Enable JSON logging") +} + +func (cfg *StatsConfig) Start(ctxt context.Context) { + + // Log with JSON format if requested + if cfg.LogJSON { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) + } + + // Enable debug if requested + if cfg.Debug { + slog.SetLogLoggerLevel(slog.LevelDebug) + } + + // Show statistics at a frequency + slog.Info("Showing statistics", "frequency", cfg.Freq, "logAll", cfg.LogAll) + cfg.showStats(ctxt) +} + +func (cfg *StatsConfig) Stop() { + close(cfg.statsChannel) + cfg.statsShutdown.Wait() +} + +func (cfg *StatsConfig) StartTask(id string) { + cfg.statsChannel <- stats{Operation: idStart, Id: id} +} + +func (cfg *StatsConfig) CancelTask(id string) { + cfg.statsChannel <- stats{Operation: idCancel, Id: id} +} + +func (cfg *StatsConfig) DoneTask(id string, bytes uint64) { + cfg.statsChannel <- stats{Operation: idDone, Id: id, Bytes: bytes} +} + +func (cfg *StatsConfig) SetStatsIdleTimeout(timeout time.Duration) { + cfg.statsChannel <- stats{Operation: idleTimeoutSet, Timeout: timeout} +} diff --git a/5-appinfra/modules/htc-infra/agent/src/stats/stats_accum.go b/5-appinfra/modules/htc-infra/agent/src/stats/stats_accum.go new file mode 100644 index 000000000..c9d3ce7b6 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/stats/stats_accum.go @@ -0,0 +1,186 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stats + +import ( + "fmt" + "math" + "time" + + "log/slog" +) + +var kbyte float64 = math.Pow(2, 10) +var mbyte float64 = math.Pow(2, 20) +var gbyte float64 = math.Pow(2, 30) + +func getFormattedDuration(t time.Duration) string { + if t >= time.Minute { + return fmt.Sprintf("%3dm ", int(t.Minutes())) + } else if t >= time.Second { + return fmt.Sprintf("%3ds ", int(t.Seconds())) + } else if t >= time.Millisecond { + return fmt.Sprintf("%3dms", t.Milliseconds()) + } else if t >= time.Microsecond { + return fmt.Sprintf("%3dμs", t.Microseconds()) + } else { + return fmt.Sprintf("%3dns", t.Nanoseconds()) + } +} + +func getFormattedBytes(bytes float64) string { + if bytes > gbyte { + return fmt.Sprintf("%3.0fGB/s", bytes/gbyte) + } else if bytes > mbyte { + return fmt.Sprintf("%3.0fMB/s", bytes/mbyte) + } else if bytes > kbyte { + return fmt.Sprintf("%3.0fKB/s", bytes/kbyte) + } else { + return fmt.Sprintf("%3.0f B/s", bytes) + } +} + +func getFormattedOps(ops float64) string { + if ops > 1_000_000_000 { + return fmt.Sprintf("%3.0fg/s", ops/1_000_000_000) + } else if ops > 1_000_000 { + return fmt.Sprintf("%3.0fm/s", ops/1_000_000) + } else if ops > 1_000 { + return fmt.Sprintf("%3.0fk/s", ops/1_000) + } else { + return fmt.Sprintf("%3.0f /s", ops) + } +} + +// Accumulated stats for a window +type accumulatedStats struct { + showHdr bool // Whether the header has been shown + + // Window start time + start time.Time // Start time for distribution + + // Tracking for completed operations in the window + timings *timeDistribution // Latency time distribution + ops uint64 // Completed operations + bytes uint64 // Transfered bytes + + // Tracking for total op time in the window + optime int64 + + // Tracking for operations (all operations outstanding) + currentOps map[string]int64 +} + +func NewAccumulatedStats() *accumulatedStats { + return &accumulatedStats{ + showHdr: false, + start: time.Now(), + timings: NewTimeDistribution(), + ops: 0, + bytes: 0, + optime: 0, + currentOps: make(map[string]int64), + } +} + +func (a *accumulatedStats) LogStats(reset bool, logJSON bool) { + + // Timing + n := time.Now() + dur := n.Sub(a.start) + + // Average latency + latencies := a.timings.GetPercentile([]float64{1.0, 0.99, 0.95, 0.5, 0.0}) + + // Show stats header if not JSON and first time + if !a.showHdr && !logJSON { + slog.Info(fmt.Sprintf("%4.4s %7.7s %6.6s %5.5s %5.5s %5.5s %5.5s %5.5s", + "Load", "Bytes", "Ops", "Max", "99th", "95th", "50th", "Min")) + a.showHdr = true + } + + // Calculate the total compute execution time since the + // start of the window, including completed operations + // *and* on-going operations + executed := a.optime + for _, opStartTime := range a.currentOps { + executed += n.UnixNano() - max(opStartTime, a.start.UnixNano()) + } + + if !logJSON { + slog.Info(fmt.Sprintf("%4.1f %s %s %s %s %s %s %s", + float64(executed)/(1e9*dur.Seconds()), + getFormattedBytes(float64(a.bytes)/dur.Seconds()), + getFormattedOps(float64(a.ops)/dur.Seconds()), + getFormattedDuration(latencies[0]), + getFormattedDuration(latencies[1]), + getFormattedDuration(latencies[2]), + getFormattedDuration(latencies[3]), + getFormattedDuration(latencies[4]))) + } else { + slog.Info("statistics", + "load", float64(executed)/(1e9*dur.Seconds()), + "bytes", a.bytes, + "ops", a.ops, + "lat_max", latencies[0], + "lat_99", latencies[1], + "lat_95", latencies[2], + "lat_50", latencies[3], + "lat_min", latencies[4], + ) + } + + // Reset all stats + a.start = n + a.ops = 0 + a.bytes = 0 + a.optime = 0 + a.timings.Clear() + +} + +func (a *accumulatedStats) ActiveOp() int { + return len(a.currentOps) +} + +func (a *accumulatedStats) StartOp(id string) { + a.currentOps[id] = time.Now().UnixNano() +} + +// Cancel the operation +// +// Note that if a stats has been produced, it will have been included +// in the load for that window. Going forward it will be excluded and no +// operations, bytes, or latency will be tracked. +func (a *accumulatedStats) CancelOp(id string) { + delete(a.currentOps, id) +} + +func (a *accumulatedStats) DoneOp(id string, bytes uint64) time.Duration { + startTime, ok := a.currentOps[id] + delete(a.currentOps, id) + if !ok { + return time.Duration(0) + } + + currentTime := time.Now().UnixNano() + + a.ops += uint64(1) + a.bytes += bytes + a.optime += currentTime - max(startTime, a.start.UnixNano()) + a.timings.Add(int32(1), time.Duration(currentTime-startTime)) + + return time.Duration(currentTime - startTime) +} diff --git a/5-appinfra/modules/htc-infra/agent/src/stats/throttle.go b/5-appinfra/modules/htc-infra/agent/src/stats/throttle.go new file mode 100644 index 000000000..b5c5765d8 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/stats/throttle.go @@ -0,0 +1,181 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stats + +import ( + "context" + "iter" + "runtime" + "sync" + "time" + + "log/slog" +) + +// Wait on a WaitGroup +func Waitable(wg *sync.WaitGroup) <-chan struct{} { + ch := make(chan struct{}) + go func() { + wg.Wait() + close(ch) + }() + return ch +} + +// Throttle a sequence +// +// This will yield results to the sequence after a certain delay specified by rate (number +// of yields per second). +// +// Ramp_steps and ramp allow for a gentle increase in the rate. Ramp defines how long +// it will take to get to the full rate, while ramp_steps the number of steps to get +// there. +// +// The context is used to short-circuit any delays, but otherwise +// not used to stop the iteration. +func Throttle[V any](ctxt context.Context, seq iter.Seq2[V, error], rate float64, ramp_steps int, ramp_duration time.Duration) iter.Seq2[V, error] { + targetNanodelay := time.Duration(0) + if rate > 0 { + targetNanodelay = time.Duration(int64(1e9 / rate)) + } else { + return seq + } + + initialTotalRampDelay := targetNanodelay + useRamp := ramp_duration > 0 && ramp_steps > 0 + if useRamp { + potentialInitialDelay := time.Duration(0) + if ramp_steps > 0 { // Prevent division by zero if ramp_steps is 0 + potentialInitialDelay = ramp_duration / time.Duration(ramp_steps) + } + + if potentialInitialDelay > targetNanodelay { + initialTotalRampDelay = potentialInitialDelay + } + // Log if ramp effect might be minimal + if initialTotalRampDelay <= targetNanodelay { + slog.Debug("Initial ramp delay is not significantly larger than target delay; ramp effect might be minimal or start at target rate.", "initial", initialTotalRampDelay, "target", targetNanodelay) + } + } + + s_next_op_time := time.Now() + rampStartTime := s_next_op_time + + return func(yield func(val V, err error) bool) { + for v, err := range seq { + if err != nil { + var empty V + if !yield(empty, err) { + return + } + continue + } + + currentOpDispatchTime := time.Now() + currentDelay := targetNanodelay + + if useRamp { + elapsedRampTime := currentOpDispatchTime.Sub(rampStartTime) + if elapsedRampTime < ramp_duration { + progressRatio := float64(elapsedRampTime) / float64(ramp_duration) + deltaDelay := initialTotalRampDelay - targetNanodelay + currentDelay = initialTotalRampDelay - time.Duration(float64(deltaDelay)*progressRatio) + + if currentDelay < targetNanodelay { + currentDelay = targetNanodelay + } + if initialTotalRampDelay > targetNanodelay && currentDelay > initialTotalRampDelay { + currentDelay = initialTotalRampDelay + } + } else { + currentDelay = targetNanodelay + useRamp = false + } + } + + if s_next_op_time.After(currentOpDispatchTime) { + sleepDuration := s_next_op_time.Sub(currentOpDispatchTime) + select { + case <-ctxt.Done(): + return + case <-time.After(sleepDuration): + } + } else { + s_next_op_time = currentOpDispatchTime + } + + s_next_op_time = s_next_op_time.Add(currentDelay) + + if !yield(v, nil) { + return + } + } + } +} + +// Run the work, applied to a sequence, with a pool of parallel workers. +// +// The first error will stop the processing of new work, and, once the workers +// are all finished, the first non-nil error is returned. +// +// If there are no errors nil is returned. +func ApplyParallel[V any](seq iter.Seq2[V, error], workers int, work func(v V) error) (err error) { + err = nil + + if workers <= 0 { + workers = runtime.NumCPU() + } + + err_ch := make(chan error) + + working := 0 + + for v, seq_err := range seq { + + // Break early on error + if seq_err != nil { + err = seq_err + break + } + + // If at limit, pull an error first. + if working == workers { + last_err := <-err_ch + if last_err != nil { + working-- + err = last_err + break + } + } else { + working++ + } + + // Dispatch the work + go func() { + err := work(v) + err_ch <- err + }() + } + + for working > 0 { + last_err := <-err_ch + working-- + if last_err != nil && err == nil { + err = last_err + } + } + + return +} diff --git a/5-appinfra/modules/htc-infra/agent/src/stats/timings.go b/5-appinfra/modules/htc-infra/agent/src/stats/timings.go new file mode 100644 index 000000000..3de200faa --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/stats/timings.go @@ -0,0 +1,85 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stats + +import ( + "log" + "sort" + "time" +) + +type timePoint struct { + timing time.Duration + weight int32 +} + +type timeDistribution struct { + points []timePoint + total int64 +} + +func NewTimeDistribution() *timeDistribution { + return &timeDistribution{ + points: make([]timePoint, 0, 10), + total: 0, + } +} + +func (t *timeDistribution) Clear() { + t.points = t.points[:0] + t.total = 0 +} + +func (t *timeDistribution) Add(weight int32, timing time.Duration) { + t.points = append(t.points, timePoint{timing, weight}) + t.total += int64(weight) +} + +// percentile should be greatest to least, no more than 1.0 (max) or less than 0.0 (min). +func (t *timeDistribution) GetPercentile(percentile []float64) []time.Duration { + + // Sort largest to smallest + sort.Slice(t.points, func(i, j int) bool { + return t.points[i].timing > t.points[j].timing + }) + + // Validate percentile + for j := 0; j < len(percentile); j++ { + if percentile[j] > 1.0 || percentile[j] < 0.0 { + log.Fatalf("invalid percentile: %f", percentile[j]) + } + if j > 0 && percentile[j-1] <= percentile[j] { + log.Fatalf("invalid percentile order") + } + } + + values := make([]time.Duration, len(percentile)) + + // Walk forwards, going backwards + total := t.total + for i, j := 0, 0; i < len(t.points) && j < len(percentile); i++ { + total -= int64(t.points[i].weight) + curr_perc := float64(total) / float64(t.total) + + // It's already at the required pecentile + for j < len(percentile) && curr_perc <= percentile[j] { + values[j] = t.points[i].timing + j++ + } + } + + return values + +} diff --git a/5-appinfra/modules/htc-infra/agent/src/stats/timings_test.go b/5-appinfra/modules/htc-infra/agent/src/stats/timings_test.go new file mode 100644 index 000000000..4828b4a21 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/stats/timings_test.go @@ -0,0 +1,53 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stats + +import ( + "testing" + "time" +) + +func TestTimings(t *testing.T) { + RunTest(t, []timePoint{ + {time.Second * 2, 1}, + {time.Second * 1, 1}, + {time.Second * 4, 1}, + {time.Second * 3, 1}, + }, + []float64{1, 0.5, 0.0}, + []time.Duration{ + time.Second * 4, + time.Second * 3, + time.Second * 1, + }) +} + +func RunTest(t *testing.T, data []timePoint, percentile []float64, expected_results []time.Duration) { + tt := NewTimeDistribution() + for _, d := range data { + tt.Add(d.weight, d.timing) + } + + results := tt.GetPercentile(percentile) + if len(expected_results) != len(results) { + t.Fatalf("expected same results length") + } + for i := 0; i < len(results); i++ { + if results[i] != expected_results[i] { + t.Fatalf("off, expected %v, got %v timing for percentile %f", + expected_results[i], results[i], percentile[i]) + } + } +} diff --git a/5-appinfra/modules/htc-infra/agent/src/test/generate.go b/5-appinfra/modules/htc-infra/agent/src/test/generate.go new file mode 100644 index 000000000..90d743641 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/test/generate.go @@ -0,0 +1,45 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" +) + +func AddTestCommands(stats *stats.StatsConfig, google *gcp.GoogleConfig) *cobra.Command { + + src := &Source{} + + genCmd := &cobra.Command{ + Use: "test", + Short: "Generate test load", + Args: cobra.ExactArgs(0), + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + stats.Start(cmd.Context()) + if err := google.Start(cmd.Context(), "controller"); err != nil { + return err + } + return nil + }, + } + src.AddGenerateFlags(genCmd, google) + + genCmd.AddCommand(NewGRPCGenerator(src, stats, google)) + genCmd.AddCommand(NewPubSubGenerator(src, stats, google)) + + return genCmd +} diff --git a/5-appinfra/modules/htc-infra/agent/src/test/grpc.go b/5-appinfra/modules/htc-infra/agent/src/test/grpc.go new file mode 100644 index 000000000..25533ad74 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/test/grpc.go @@ -0,0 +1,60 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "fmt" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" + "google.golang.org/protobuf/proto" +) + +func NewGRPCGenerator(src *Source, stats *stats.StatsConfig, google *gcp.GoogleConfig) *cobra.Command { + + cfg := &protoio.BackendConfiguration{} + + cmd := &cobra.Command{ + Use: "grpc", + Short: "gRPC roundtrip and throughput latency tests", + Args: cobra.ExactArgs(0), + } + cfg.Initialize(cmd, google) + cmd.RunE = func(c *cobra.Command, args []string) error { + + inputType, _, err := cfg.GetTypes(c.Context()) + if err != nil { + return err + } + + invoker, err := cfg.GetProtoInvoker(c.Context(), stats) + if err != nil { + return err + } + + return src.StartSourceProto(c.Context(), inputType, func(ctxt context.Context, msg proto.Message, cnt int) error { + _, err := invoker(ctxt, msg) + if err != nil { + return fmt.Errorf("failed calling grpc: %w", err) + } + return nil + }) + } + + return cmd +} diff --git a/5-appinfra/modules/htc-infra/agent/src/test/pubsub.go b/5-appinfra/modules/htc-infra/agent/src/test/pubsub.go new file mode 100644 index 000000000..ec15a2665 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/test/pubsub.go @@ -0,0 +1,233 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "regexp" + + "cloud.google.com/go/pubsub" + + "log/slog" + + "google.golang.org/protobuf/reflect/protoreflect" + + "fmt" + "time" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" +) + +func NewPubSubGenerator(src *Source, stats *stats.StatsConfig, google *gcp.GoogleConfig) *cobra.Command { + + jsonPubSub := true + maxMessagesOutstanding := 250000 + cmd := &cobra.Command{ + Use: "pubsub [subscription]", + Short: "Pubsub roundtrip throughput and latency tests", + Args: cobra.RangeArgs(1, 2), + } + cmd.Flags().BoolVar(&jsonPubSub, "jsonPubSub", jsonPubSub, "Enable JSON in Pub/Sub instead of byte-encoded protobuf") + cmd.Flags().IntVar(&maxMessagesOutstanding, "maxMessagesOutstanding", maxMessagesOutstanding, "Maximum messages outstanding on publish") + + cmd.RunE = func(c *cobra.Command, args []string) error { + slog.Info("Statistics are for messages (ops/second = messages per second).") + + // Create the client + client, err := google.PubSubClient(c.Context()) + if err != nil { + return fmt.Errorf("failed creating the client: %w", err) + } + + // Job Number is the unix timestamp + jobNum := time.Now().Unix() + + // Start the subscription + if len(args) == 2 { + slog.Debug("Subscribing", "project", google.ProjectID, "subscription", args[1]) + sub := client.SubscriptionInProject(args[1], google.ProjectID) + sub.ReceiveSettings.MaxOutstandingBytes = -1 + sub.ReceiveSettings.MaxOutstandingMessages = -1 + + go func() { + msgReceiver := getMessageReceiver(stats, jobNum) + if err := sub.Receive(c.Context(), msgReceiver); err != nil { + slog.Warn("Subscription error", "error", err) + } + }() + } + + // Start the source of test data + topic := client.Topic(args[0]) + if maxMessagesOutstanding > 0 { + topic.PublishSettings.FlowControlSettings.MaxOutstandingMessages = maxMessagesOutstanding + topic.PublishSettings.FlowControlSettings.LimitExceededBehavior = pubsub.FlowControlBlock + } + publishOp := getMessagePublisher(stats, topic, jobNum) + + // Start publishing + slog.Debug("Publishing", "project", google.ProjectID, "topic", args[0]) + var desc protoreflect.MessageDescriptor + if !jsonPubSub { + desc, err = getMessageDescriptor(c.Context(), client, args[0]) + if err != nil { + return fmt.Errorf("failed finding schema from topic '%s': %w", args[0], err) + } + } + if err := src.StartSourceBytes(c.Context(), desc, publishOp); err != nil { + return fmt.Errorf("error publishing: %v", err) + } + + // Cancel as soon as we are idle (no more outstanding requests + stats.SetStatsIdleTimeout(time.Duration(0)) + + // Wait for cancellation + <-c.Context().Done() + slog.Info("All sent messages received") + + return nil + } + + return cmd +} + +func getMessageReceiver(stats *stats.StatsConfig, jobNum int64) func(ctxt context.Context, msg *pubsub.Message) { + + return func(ctxt context.Context, msg *pubsub.Message) { + + // Debug + slog.Debug("Receiving response", "response", msg) + + // Always ack for received messages + r := msg.AckWithResult() + + // If exactly once enabled and there is an ack reslut, + // block until the result is returned and a pubsub.AcknowledgeStatus + // is returned for the acked message. + if r != nil { + status, err := r.Get(ctxt) + if err != nil { + slog.Warn("Failed when calling result.Get", "error", err, "msg", msg.ID) + } + if status != pubsub.AcknowledgeStatusSuccess { + slog.Warn("Message acknowledged failed", "status", status, "id", msg.ID, "attributes", msg.Attributes) + } + } + + // Record performance of this single message and size of the message + srcId, ok := msg.Attributes["srcId"] + if ok { + stats.DoneTask(srcId, uint64(len(msg.Data))) + } + } +} + +func getMessagePublisher(stats *stats.StatsConfig, pubTopic *pubsub.Topic, jobNum int64) func(ctxt context.Context, msg []byte, cnt int) error { + return func(ctxt context.Context, msg []byte, cnt int) error { + slog.Debug("Publishing request", "count", cnt, "topic", pubTopic.ID()) + + // Stop if the context is cancelled + if ctxt.Err() != nil { + return ctxt.Err() + } + + // Publish message + srcId := fmt.Sprintf("Job-%d-Msg-%d", jobNum, cnt) + stats.StartTask(srcId) + slog.Debug("publishing", + "srcId", srcId, + "data", msg) + res := pubTopic.Publish(ctxt, &pubsub.Message{ + Data: msg, + Attributes: map[string]string{ + "srcTimeNano": fmt.Sprintf("%d", time.Now().UnixNano()), + "srcId": srcId, + }, + }) + + // Log errors + go func() { + _, err := res.Get(ctxt) + if err != nil { + + // It failed sending, no need to wait for it now + stats.CancelTask(srcId) + + // Log if it's a general error, not cancellation + if err != context.Canceled { + slog.Warn("error publishing", "error", err) + } + } + }() + + return nil + } +} + +var SCHEMA_PATTERN = regexp.MustCompile(`^projects/([^/]*)/schemas/([^/]*)$`) + +func getMessageDescriptor(ctxt context.Context, client *pubsub.Client, topicName string) (protoreflect.MessageDescriptor, error) { + topic := client.Topic(topicName) + defer topic.Stop() + + exists, err := topic.Exists(ctxt) + if err != nil { + return nil, fmt.Errorf("error looking up topic '%s': %w", topicName, err) + } + if !exists { + return nil, fmt.Errorf("topic %s does not exist", topic) + } + + topicCfg, err := topic.Config(ctxt) + if err != nil { + return nil, fmt.Errorf("failed finding topic configuration: %w", err) + } + + if topicCfg.SchemaSettings == nil || topicCfg.SchemaSettings.Schema == "" { + return nil, fmt.Errorf("topic has no schema associated") + } + + if topicCfg.SchemaSettings.Encoding != pubsub.EncodingBinary { + return nil, fmt.Errorf("only support binary encoding") + } + + schema_match := SCHEMA_PATTERN.FindStringSubmatch(topicCfg.SchemaSettings.Schema) + if schema_match == nil { + return nil, fmt.Errorf("invalid schema name: %s", topicCfg.SchemaSettings.Schema) + } + + schemaClient, err := pubsub.NewSchemaClient(ctxt, schema_match[1]) + if err != nil { + return nil, fmt.Errorf("error creating schema client: %w", err) + } + + schemaConfig, err := schemaClient.Schema(ctxt, schema_match[2], pubsub.SchemaViewFull) + if err != nil { + return nil, fmt.Errorf("error finding schema '%s': %w", topicCfg.SchemaSettings.Schema, err) + } + + if schemaConfig.Name != topicCfg.SchemaSettings.Schema { + return nil, fmt.Errorf("not matching name") + } + + if schemaConfig.Type != pubsub.SchemaProtocolBuffer { + return nil, fmt.Errorf("must be a protocolbuffer schema") + } + + return protoio.CompileDescriptor(ctxt, schemaConfig.Definition) +} diff --git a/5-appinfra/modules/htc-infra/agent/src/test/source.go b/5-appinfra/modules/htc-infra/agent/src/test/source.go new file mode 100644 index 000000000..7f20335f9 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/src/test/source.go @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "iter" + "sync/atomic" + "time" + + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/spf13/cobra" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +type Source struct { + Source string + Loop bool + Buffer int + + Ramp time.Duration + RampSteps int + MaxBatch int + Rate float64 + + // Private + google *gcp.GoogleConfig +} + +func (s *Source) AddGenerateFlags(cmd *cobra.Command, google *gcp.GoogleConfig) { + s.google = google + + cmd.PersistentFlags().StringVar(&s.Source, "source", s.Source, "Source of data") + cmd.PersistentFlags().BoolVar(&s.Loop, "loop", s.Loop, "Loop over data repeatedly") + cmd.PersistentFlags().IntVar(&s.Buffer, "buffer", s.Buffer, "Size of buffer for reading") + cmd.PersistentFlags().IntVar(&s.MaxBatch, "max_batch", s.MaxBatch, "Maximum batch operations outstanding (parallel operations, 0 => serial)") + cmd.PersistentFlags().Float64Var(&s.Rate, "rate", s.Rate, "Batch operations per second max (0 = no limit)") + cmd.PersistentFlags().DurationVar(&s.Ramp, "ramp", s.Ramp, "Duration for ramp of rate (0 = no ramp)") + cmd.PersistentFlags().IntVar(&s.RampSteps, "ramp_steps", s.RampSteps, "Number of steps for ramping") +} + +func StartSource[T any](s *Source, ctxt context.Context, op func(ctxt context.Context, v T, cnt int) error, seqf func() iter.Seq2[T, error]) error { + + // Loop forever if requested + var seq iter.Seq2[T, error] + if s.Loop { + seq = func(yield func(t T, e error) bool) { + var empty T + for ctxt.Err() == nil { + iseq := seqf() + for v, e := range iseq { + if e != nil { + yield(empty, e) + return + } + if !yield(v, nil) { + return + } + } + } + } + } else { + seq = seqf() + } + + // If there is a rate, then limit the rate + if s.Rate > 0 { + seq = stats.Throttle[T](ctxt, seq, s.Rate, s.RampSteps, s.Ramp) + } + + // Always one worker if none specified + if s.MaxBatch == 0 { + s.MaxBatch = 1 + } + + // Counter starts at one + var cnt atomic.Int64 + cnt.Store(1) + + // Apply in parallel + return stats.ApplyParallel[T](seq, s.MaxBatch, func(v T) error { + prevcnt := cnt.Add(1) + return op(ctxt, v, int(prevcnt)) + }) +} + +func (s *Source) StartSourceBytes(ctxt context.Context, desc protoreflect.MessageDescriptor, op func(context.Context, []byte, int) error) error { + return StartSource[[]byte](s, ctxt, op, func() iter.Seq2[[]byte, error] { + seq := protoio.ReadLines(ctxt, s.google, s.Source) + if desc != nil { + seq = protoio.MapIterErr(seq, protoio.JSONToProtoBytes(desc)) + } + return seq + }) +} + +func (s *Source) StartSourceProto(ctxt context.Context, desc protoreflect.MessageDescriptor, op func(context.Context, proto.Message, int) error) error { + return StartSource[proto.Message](s, ctxt, op, func() iter.Seq2[proto.Message, error] { + return protoio.MapIterErr(protoio.ReadLines(ctxt, s.google, s.Source), protoio.JSONToProto(desc)) + }) +} diff --git a/5-appinfra/modules/htc-infra/american-option/.gitignore b/5-appinfra/modules/htc-infra/american-option/.gitignore new file mode 100644 index 000000000..9e317734c --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/.gitignore @@ -0,0 +1,6 @@ +# Protobuf generated python files +*_pb2*.py + +# Generate python files +__pycache__/** +.venv/** diff --git a/5-appinfra/modules/htc-infra/american-option/Dockerfile b/5-appinfra/modules/htc-infra/american-option/Dockerfile new file mode 100644 index 000000000..c9517594e --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/Dockerfile @@ -0,0 +1,28 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.12 + +# Install applications and dependencies +WORKDIR /app +COPY *.proto setup.sh requirements.txt /app/ + +# Setup virtual environment +RUN /app/setup.sh + +# Copy in rest of source +COPY *.py *.sh /app/ + +# Entrypoint +ENTRYPOINT [ "/app/.venv/bin/python3", "main.py" ] diff --git a/5-appinfra/modules/htc-infra/american-option/README.md b/5-appinfra/modules/htc-infra/american-option/README.md new file mode 100644 index 000000000..07fefa737 --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/README.md @@ -0,0 +1,98 @@ + +# American Option Example Library + +## Overview + +This is an example Python script using [Quantlib](https://www.quantlib.org/) in order +to calculate American Option prices. This is provided purely as an example and is not +intended to be used as-is for financial pricing. + +## Setup + +There is a provided [Dockerfile](Dockerfile) and a [setup script](setup.sh) to +configure a virtual environment and generate protobuf code. + +You can run `setup.sh` locally for development purposes and it can also be used +in the `Dockerfile`. + +## Running locally + +### Setup the virtual environment + +Run setup.sh: + +```sh +./setup.sh +``` + +### Generate test data + +Generate 1000 example tasks: + +```sh +.venv/bin/python3 main.py gentasks --count 1000 tasks.jsonl +``` + +### With the file directly + +Directly run pricer with the test data: + +```sh +.venv/bin/python3 main.py load tasks.jsonl +``` + +### With gRPC + +Run the pricing engine locally on the default port 2002: + +```sh +.venv/bin/python3 main.py serve +``` + +In a separate command prompt, [grpcurl](https://github.com/fullstorydev/grpcurl) with the test data to try it out: +to generate JSON test data for the pricing library. This will only send in the +first request. + +```sh +head -n 1 tasks.jsonl | grpcurl -d @ -plaintext localhost:2002 main.PricingService/CalcPrices +``` + +## Running in a container + +### Build the container + +```sh +docker build -t pricer . +``` + +### Generate test data + +Generate 1000 example tasks: + +```sh +docker run pricer gentasks --count 1000 - > tasks.jsonl +``` + +### With the file directly + +Directly run pricer with the test data: + +```sh +docker run -v $PWD:/data pricer load /data/tasks.jsonl +``` + +### With gRPC + +Run the pricing engine locally on the default port 2002: + +```sh +docker run -p 2002:2002 pricer serve +``` + +In a separate command prompt, [grpcurl](https://github.com/fullstorydev/grpcurl) with the test data to try it out: +to generate JSON test data for the pricing library. This will only send in the +first request. + +```sh +head -n 1 tasks.jsonl | grpcurl -d @ -plaintext localhost:2002 main.PricingService/CalcPrices +``` diff --git a/5-appinfra/modules/htc-infra/american-option/american_option.py b/5-appinfra/modules/htc-infra/american-option/american_option.py new file mode 100755 index 000000000..358613b8d --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/american_option.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# flake8: noqa + +import logging + +from request_pb2 import PricingRequest +from response_pb2 import PricingResponse + +import QuantLib as ql + + +MarketData = PricingRequest.MarketData +Decimal = PricingRequest.Decimal +Date = PricingRequest.Date + +logger = logging.getLogger(__name__) + + +def to_dbl(decimal: PricingRequest.Decimal) -> float: + _NANO = 1e-9 + return decimal.units + decimal.nanos * _NANO + + +def to_date(date: PricingRequest.Date): + return ql.Date(date.day, date.month, date.year) + + +def to_ccy(ccy: PricingRequest.Currency) -> str: + return PricingRequest.Currency.Name(ccy) + + +def get_options(refdate, opts: list[MarketData.EquityOption]): + out_opts = {} + for opt in opts: + ccy = to_ccy(opt.currency) + + strikes = [to_dbl(p) for p in opt.strike_prices] + expirations = [to_date(d) for d in opt.strike_dates] + + volMatrix = ql.Matrix(len(strikes), len(expirations)) + for i in range(len(strikes)): + for j in range(len(expirations)): + volMatrix[i][j] = to_dbl(opt.implied_vols[j * len(strikes) + i]) + + volatilitySurface = ql.BlackVarianceSurface( + refdate, ql.TARGET(), expirations, strikes, volMatrix, ql.Actual365Fixed() + ) + volatilitySurface.enableExtrapolation() + + out_opts[opt.id] = { + "quote": ql.SimpleQuote(to_dbl(opt.spot_price)), + "ccy": ccy, + "surface": volatilitySurface, + } + + return out_opts + + +def get_riskfree_curves( + refdate, key, curves: list[PricingRequest.MarketData.RateCurve] +): + out_curves = {} + for curve in curves: + if curve.rate_type != key: + continue + ccy = to_ccy(curve.currency) + dates = [refdate] + [to_date(rd.date) for rd in curve.discounts] + discounts = [1.0] + [to_dbl(rd.value) for rd in curve.discounts] + out_curves[ccy] = ql.DiscountCurve(dates, discounts, ql.Actual360()) + + return out_curves + + +def quantlib_run(req: PricingRequest) -> PricingResponse: + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Processing request:\n{req}") + + # Set reference date + refDate = to_date(req.marketdata.reference_date) + ql.Settings.instance().evaluationDate = refDate + + # Fetch risk free curve + curves = get_riskfree_curves( + refDate, + PricingRequest.MarketData.RateType.RISK_FREE_CURVE, + req.marketdata.rate_curves, + ) + + # Fetch option volatilities + vols = get_options(refDate, req.marketdata.equity_options) + + results = [] + for opt_req in req.american_option_request: + ccy = to_ccy(opt_req.currency) + exercise = ql.AmericanExercise(refDate, to_date(opt_req.expiry_date)) + + # Contract prices + payoff = ql.PlainVanillaPayoff( + ql.Option.Call if opt_req.is_call_option else ql.Option.Put, + to_dbl(opt_req.strike), + ) + option = ql.VanillaOption(payoff, exercise) + + if opt_req.equity not in vols: + raise Exception(f"no marketdata for {opt_req.equity}") + + volatility = vols[opt_req.equity] + + if ccy not in curves: + raise Exception(f"no curve for currency {ccy}") + + if volatility["ccy"] != ccy: + raise Exception(f"ccy of equity is {volatility.ccy} vs {ccy}") + + # No dividends + dividendYield = ql.FlatForward(refDate, 0.00, ql.Actual365Fixed()) + + # From marketdata + process = ql.BlackScholesMertonProcess( + ql.QuoteHandle(volatility["quote"]), + ql.YieldTermStructureHandle(dividendYield), + ql.YieldTermStructureHandle(curves[ccy]), + ql.BlackVolTermStructureHandle(volatility["surface"]), + ) + + results = [] + option.setPricingEngine(ql.BaroneAdesiWhaleyApproximationEngine(process)) + results.append(option.NPV()) + + response = PricingResponse(value=results) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Returning response:\n{response}") + + return response diff --git a/5-appinfra/modules/htc-infra/american-option/main.py b/5-appinfra/modules/htc-infra/american-option/main.py new file mode 100755 index 000000000..6cedbdbcd --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/main.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# flake8: noqa + +from concurrent import futures +import argparse +import sys +import gzip +import grpc +import logging + +from request_pb2 import PricingRequest +import service_pb2_grpc +import service_pb2 +from grpc_reflection.v1alpha import reflection + +from american_option import quantlib_run +from google.protobuf.json_format import Parse, MessageToJson + +from test_data import generate_test_data + + +logger = logging.getLogger(__name__) + + +def process_file(input): + # Get the output file (compress if needed) + if input.name.endswith(".gz"): + input = gzip.open(input, mode="rb") + + with input as f: + for line in f: + # Parse the request from the line + req = PricingRequest() + Parse(line, req) + + # Print out the result + print( + MessageToJson( + quantlib_run(req), + indent=None, + ) + ) + + +class Pricer(service_pb2_grpc.PricingServiceServicer): + def CalcPrices(self, request, context): + return quantlib_run(request) + + +def serve(port): + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=10), maximum_concurrent_rpcs=2 + ) + service_pb2_grpc.add_PricingServiceServicer_to_server(Pricer(), server) + SERVICE_NAMES = ( + service_pb2.DESCRIPTOR.services_by_name["PricingService"].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port(f"[::]:{port}") + server.start() + server.wait_for_termination() + + +def main(args): + parser = argparse.ArgumentParser( + prog="calc", description="Calculate American Options." + ) + parser.add_argument("-d", "--debug", help="Debugging enabled", action="store_true") + parser.add_argument( + "--logJSON", help="Log to JSON (currently unsupported)", action="store_true" + ) + + subparsers = parser.add_subparsers(dest="command_name") + + load_parser = subparsers.add_parser(name="load", help="Load from file") + load_parser.add_argument( + "load_file", help="Input JSONL file to process", type=argparse.FileType("rb") + ) + + serve_parser = subparsers.add_parser(name="serve", help="Serve over gRPC") + serve_parser.add_argument( + "-p", "--port", help="Port to wait on", default=2002, type=int + ) + + gen_parser = subparsers.add_parser( + name="gentasks", help="Generate option pricing requests" + ) + gen_parser.add_argument( + "-c", + "--count", + help="Number of output records to generate", + type=int, + default=1, + ) + gen_parser.add_argument( + "outfile", help="Output file with JSONL records", type=argparse.FileType("wb") + ) + + args = parser.parse_args() + + # Startup logging + logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) + if args.logJSON: + logger.info("Logging JSON formatted currently not supported.") + + logger.info("Starting American Option example") + + if args.command_name == "load": + logger.info(f"Processing input file {args.load_file.name}") + process_file(args.load_file) + elif args.command_name == "serve": + logger.info(f"Starting gRPC Server on port {args.port}") + serve(args.port) + elif args.command_name == "gentasks": + logger.info(f"Generating {args.count} output to {args.outfile.name}") + generate_test_data(args.outfile, args.count) + else: + parser.print_help() + + +if __name__ == "__main__": + main(sys.argv) diff --git a/5-appinfra/modules/htc-infra/american-option/request.proto b/5-appinfra/modules/htc-infra/american-option/request.proto new file mode 100644 index 000000000..52cc25cb5 --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/request.proto @@ -0,0 +1,149 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package main; + +// Interest rate swap proto +message PricingRequest { + + message Date { + int32 year = 1; + int32 month = 2; + int32 day = 3; + } + + enum Currency { + CURRENCY_UNDEFINED = 0; + + // G10 Currencies. + USD = 1; // US dollar + GBP = 2; // British Pound + EUR = 3; // Euro + } + + + // Represents decimal numbers exactly. Protobufs don't have native support for + // decimals. A commonly used pattern to represent decimals is shown here: + // https://github.com/googleapis/googleapis/blob/master/google/type/money.proto + // This is also a pattern recommended here: + // https://docs.microsoft.com/en-us/dotnet/architecture/grpc-for-wcf-developers/protobuf-data-types#decimals + message Decimal { + int64 units = 1; + sfixed32 nanos = 2; + } + + + // Align with Quantlib + // From https://quantlib-python-docs.readthedocs.io/en/latest/dates.html#weekday-correction + + // Business day convention proto. Determines how non-business days are treated + // if the payment is due on a non-business + enum BusinessDayConvention { + NONE = 0; + FOLLOWING = 1; + MODIFIED_FOLLOWING = 2; + PRECEDING = 3; + MODIFIED_PRECEDING = 4; + } + + + // Align with Quantlib + // From https://quantlib-python-docs.readthedocs.io/en/latest/dates.html#ql.Calendar.holidayList + + // Bank holidays captures the calendar used to generate public holidays. + enum BankHolidays { + BANK_HOLIDAYS_UNKNOWN = 0; + US = 1; + UK = 2; + } + + + // Holds metadata for an instrument. + message InstrumentMetadata { + // The type of the instrument. This should be the name of the containing + // message (e.g. ForwardRateAgreement). + string instrument_type = 1; + // An arbitrary identifier for the instrument/trade. This ID should be + // immutable once assigned to an instrument - in particular, aging events + // shouldn't change this ID. + string id = 2; + // Counterparty to this trade. + string counterparty = 3; + } + + + message AmericanOptionRequest { + bool short_position = 1; + Date expiry_date = 2; + Currency currency = 3; + string equity = 4; + Decimal contract_amount = 5; + Decimal strike = 6; + BusinessDayConvention business_day_convention = 8; + BankHolidays bank_holidays = 9; + int32 settlement_days = 10; + InstrumentMetadata metadata = 11; + bool is_call_option = 12; + } + + + // MarketData protobuf definition + message MarketData { + message RateDiscount { + Date date = 1; + Decimal value = 2; + } + + message EquityOption { + string id = 1; + Currency currency = 2; + Decimal spot_price = 3; + + // Strike details + repeated Date strike_dates = 4; + repeated Decimal strike_prices = 5; + + // Single array that is len(dates) * len(strike_prices) + repeated Decimal implied_vols = 6; + } + + enum RateType { + RATE_UNDEFINED = 0; + RISK_FREE_CURVE = 1; + } + + message RateCurve { + RateType rate_type = 1; + Currency currency = 2; + repeated RateDiscount discounts = 3; + } + + Date reference_date = 1; + + repeated RateCurve rate_curves = 2; + repeated EquityOption equity_options = 3; + } + + + // Pricing requests + repeated AmericanOptionRequest american_option_request = 1; + + // Source of Marketdata + oneof marketdata_source { + MarketData marketdata = 100; + string marketdata_ref_id = 101; + } +} diff --git a/5-appinfra/modules/htc-infra/american-option/requirements.in b/5-appinfra/modules/htc-infra/american-option/requirements.in new file mode 100644 index 000000000..6fdac8e9f --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/requirements.in @@ -0,0 +1,19 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +quantlib +grpcio-reflection +# Due to https://github.com/grpc/grpc/issues/37609 these need to be sync'd +grpcio-tools==1.66.1 +protobuf==5.27.2 diff --git a/5-appinfra/modules/htc-infra/american-option/requirements.txt b/5-appinfra/modules/htc-infra/american-option/requirements.txt new file mode 100644 index 000000000..3cb3e67ba --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/requirements.txt @@ -0,0 +1,162 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes ./requirements.in +grpcio==1.66.1 \ + --hash=sha256:0e6c9b42ded5d02b6b1fea3a25f036a2236eeb75d0579bfd43c0018c88bf0a3e \ + --hash=sha256:161d5c535c2bdf61b95080e7f0f017a1dfcb812bf54093e71e5562b16225b4ce \ + --hash=sha256:17663598aadbedc3cacd7bbde432f541c8e07d2496564e22b214b22c7523dac8 \ + --hash=sha256:1c17ebcec157cfb8dd445890a03e20caf6209a5bd4ac5b040ae9dbc59eef091d \ + --hash=sha256:292a846b92cdcd40ecca46e694997dd6b9be6c4c01a94a0dfb3fcb75d20da858 \ + --hash=sha256:2ca2559692d8e7e245d456877a85ee41525f3ed425aa97eb7a70fc9a79df91a0 \ + --hash=sha256:307b1d538140f19ccbd3aed7a93d8f71103c5d525f3c96f8616111614b14bf2a \ + --hash=sha256:30a1c2cf9390c894c90bbc70147f2372130ad189cffef161f0432d0157973f45 \ + --hash=sha256:31a049daa428f928f21090403e5d18ea02670e3d5d172581670be006100db9ef \ + --hash=sha256:35334f9c9745add3e357e3372756fd32d925bd52c41da97f4dfdafbde0bf0ee2 \ + --hash=sha256:3750c5a00bd644c75f4507f77a804d0189d97a107eb1481945a0cf3af3e7a5ac \ + --hash=sha256:3885f037eb11f1cacc41f207b705f38a44b69478086f40608959bf5ad85826dd \ + --hash=sha256:4573608e23f7e091acfbe3e84ac2045680b69751d8d67685ffa193a4429fedb1 \ + --hash=sha256:4825a3aa5648010842e1c9d35a082187746aa0cdbf1b7a2a930595a94fb10fce \ + --hash=sha256:4877ba180591acdf127afe21ec1c7ff8a5ecf0fe2600f0d3c50e8c4a1cbc6492 \ + --hash=sha256:48b0d92d45ce3be2084b92fb5bae2f64c208fea8ceed7fccf6a7b524d3c4942e \ + --hash=sha256:4d813316d1a752be6f5c4360c49f55b06d4fe212d7df03253dfdae90c8a402bb \ + --hash=sha256:5dd67ed9da78e5121efc5c510f0122a972216808d6de70953a740560c572eb44 \ + --hash=sha256:6f914386e52cbdeb5d2a7ce3bf1fdfacbe9d818dd81b6099a05b741aaf3848bb \ + --hash=sha256:7101db1bd4cd9b880294dec41a93fcdce465bdbb602cd8dc5bd2d6362b618759 \ + --hash=sha256:7e06aa1f764ec8265b19d8f00140b8c4b6ca179a6dc67aa9413867c47e1fb04e \ + --hash=sha256:84ca1be089fb4446490dd1135828bd42a7c7f8421e74fa581611f7afdf7ab761 \ + --hash=sha256:8a1e224ce6f740dbb6b24c58f885422deebd7eb724aff0671a847f8951857c26 \ + --hash=sha256:97ae7edd3f3f91480e48ede5d3e7d431ad6005bfdbd65c1b56913799ec79e791 \ + --hash=sha256:9c9bebc6627873ec27a70fc800f6083a13c70b23a5564788754b9ee52c5aef6c \ + --hash=sha256:a013c5fbb12bfb5f927444b477a26f1080755a931d5d362e6a9a720ca7dbae60 \ + --hash=sha256:a66fe4dc35d2330c185cfbb42959f57ad36f257e0cc4557d11d9f0a3f14311df \ + --hash=sha256:a92c4f58c01c77205df6ff999faa008540475c39b835277fb8883b11cada127a \ + --hash=sha256:aa8ba945c96e73de29d25331b26f3e416e0c0f621e984a3ebdb2d0d0b596a3b3 \ + --hash=sha256:b0aa03d240b5539648d996cc60438f128c7f46050989e35b25f5c18286c86734 \ + --hash=sha256:b1b24c23d51a1e8790b25514157d43f0a4dce1ac12b3f0b8e9f66a5e2c4c132f \ + --hash=sha256:b7ffb8ea674d68de4cac6f57d2498fef477cef582f1fa849e9f844863af50083 \ + --hash=sha256:b9feb4e5ec8dc2d15709f4d5fc367794d69277f5d680baf1910fc9915c633524 \ + --hash=sha256:bff2096bdba686019fb32d2dde45b95981f0d1490e054400f70fc9a8af34b49d \ + --hash=sha256:c30aeceeaff11cd5ddbc348f37c58bcb96da8d5aa93fed78ab329de5f37a0d7a \ + --hash=sha256:c9f80f9fad93a8cf71c7f161778ba47fd730d13a343a46258065c4deb4b550c0 \ + --hash=sha256:cfd349de4158d797db2bd82d2020554a121674e98fbe6b15328456b3bf2495bb \ + --hash=sha256:d0cd7050397b3609ea51727b1811e663ffda8bda39c6a5bb69525ef12414b503 \ + --hash=sha256:d639c939ad7c440c7b2819a28d559179a4508783f7e5b991166f8d7a34b52815 \ + --hash=sha256:e3ba04659e4fce609de2658fe4dbf7d6ed21987a94460f5f92df7579fd5d0e22 \ + --hash=sha256:ecfe735e7a59e5a98208447293ff8580e9db1e890e232b8b292dc8bd15afc0d2 \ + --hash=sha256:ef82d361ed5849d34cf09105d00b94b6728d289d6b9235513cb2fcc79f7c432c \ + --hash=sha256:f03a5884c56256e08fd9e262e11b5cfacf1af96e2ce78dc095d2c41ccae2c80d \ + --hash=sha256:f1fe60d0772831d96d263b53d83fb9a3d050a94b0e94b6d004a5ad111faa5b5b \ + --hash=sha256:f517fd7259fe823ef3bd21e508b653d5492e706e9f0ef82c16ce3347a8a5620c \ + --hash=sha256:fdb14bad0835914f325349ed34a51940bc2ad965142eb3090081593c6e347be9 + # via + # grpcio-reflection + # grpcio-tools +grpcio-reflection==1.66.1 \ + --hash=sha256:bbac3c644d34647fbbec9696904008ba3eef354c2168315fe27a8a52ae79dec6 \ + --hash=sha256:dbc39d2ccce82bf13046b145b97bc9542b3b26419a848ae64a5b72634ded14ca + # via -r ./requirements.in +grpcio-tools==1.66.1 \ + --hash=sha256:0067e79b6001560ac6acc78cca11fd3504fa27f8af46e3cdbac2f4998505e597 \ + --hash=sha256:016fa273dc696c9d8045091ac50e000bce766183a6b150801f51c2946e33dbe3 \ + --hash=sha256:066648543f786cb74b1fef5652359952455dbba37e832642026fd9fd8a219b5f \ + --hash=sha256:097a069e7c640043921ecaf3e88d7af78ccd40c25dbddc91db2a4a2adbd0393d \ + --hash=sha256:0a86398a4cd0665bc7f09fa90b89bac592c959d2c895bf3cf5d47a98c0f2d24c \ + --hash=sha256:1b4acb53338072ab3023e418a5c7059cb15686abd1607516fa1453406dd5f69d \ + --hash=sha256:1ec9f4f964f8e8ed5e9cc13deb678c83d5597074c256805373220627833bc5ad \ + --hash=sha256:2226ff8d3ecba83b7622946df19d6e8e15cb52f761b8d9e2f807b228db5f1b1e \ + --hash=sha256:222d8dc218560698e1abf652fb47e4015994ec7a265ef46e012fd9c9e77a4d6b \ + --hash=sha256:23cad65ff22459aa387f543d293f54834c9aac8f76fb7416a7046556df75b567 \ + --hash=sha256:2f4b1498cb8b422fbae32a491c9154e8d47650caf5852fbe6b3b34253e824343 \ + --hash=sha256:3198815814cdd12bdb69b7580d7770a4ad4c8b2093e0bd6b987bc817618e3eec \ + --hash=sha256:3acce426f5e643de63019311171f4d31131da8149de518716a95c29a2c12dd38 \ + --hash=sha256:3d17a27c567a5e4d18f487368215cb51b43e2499059fd6113b92f7ae1fee48be \ + --hash=sha256:4df167e67b083f96bc277032a526f6186e98662aaa49baea1dfb8ecfe26ce117 \ + --hash=sha256:5055ffe840ea8f505c30378be02afb4dbecb33480e554debe10b63d6b2f641c3 \ + --hash=sha256:56e17a11f34df252b4c6fb8aa8cd7b44d162dba9f3333be87ddf7c8bf496622a \ + --hash=sha256:5b4fc56abeafae74140f5da29af1093e88ce64811d77f1a81c3146e9e996fb6a \ + --hash=sha256:5daceb9716e31edc0e1ba0f93303785211438c43502edddad7a919fc4cb3d664 \ + --hash=sha256:5f1f04578b72c281e39274348a61d240c48d5321ba8d7a8838e194099ecbc322 \ + --hash=sha256:66f527a1e3f063065e29cf6f3e55892434d13a5a51e3b22402e09da9521e98a3 \ + --hash=sha256:68d9390bf9ba863ac147fc722d6548caa587235e887cac1bc2438212e89d1de7 \ + --hash=sha256:739c53571130b359b738ac7d6d0a1f772e15779b66df7e6764bee4071cd38689 \ + --hash=sha256:796620fc41d3fbb566d9614ef22bc55df67fac1f1e19c1e0fb6ec48bc9b6a44b \ + --hash=sha256:7d789bfe53fce9e87aa80c3694a366258ce4c41b706258e9228ed4994832b780 \ + --hash=sha256:7fc3f62494f238774755ff90f0e66a93ac7972ea1eb7180c45acf4fd53b25cca \ + --hash=sha256:869b6960d5daffda0dac1a474b44144f0dace0d4336394e499c4f400c5e2f8d9 \ + --hash=sha256:88e04b7546101bc79c868c941777efd5088063a9e4f03b4d7263dde796fbabf7 \ + --hash=sha256:93d2d9e14e81affdc63d67c42eb16a8da1b6fecc16442a703ca60eb0e7591691 \ + --hash=sha256:95c44a265ff01fd05166edae9350bc2e7d1d9a95e8f53b8cd04d2ae0a588c583 \ + --hash=sha256:9a07e24feb7472419cf70ebbb38dd4299aea696f91f191b62a99b3ee9ff03f89 \ + --hash=sha256:b8660401beca7e3af28722439e07b0bcdca80b4a68f5a5a1138ae7b7780a6abf \ + --hash=sha256:b962a8767c3c0f9afe92e0dd6bb0b2305d35195a1053f84d4d31f585b87557ed \ + --hash=sha256:d19d47744c30e6bafa76b3113740e71f382d75ebb2918c1efd62ebe6ba7e20f9 \ + --hash=sha256:d4dd2ff982c1aa328ef47ce34f07af82f1f13599912fb1618ebc5fe1e14dddb8 \ + --hash=sha256:d761dfd97a10e4aae73628b5120c64e56f0cded88651d0003d2d80e678c3e7c9 \ + --hash=sha256:d8616773126ec3cdf747b06a12e957b43ac15c34e4728def91fa67249a7c689a \ + --hash=sha256:da9b0c08dbbf07535ee1b75a22d0acc5675a808a3a3df9f9b21e0e73ddfbb3a9 \ + --hash=sha256:df1a174a6f9d3b4c380f005f33352d2e95464f33f021fb08084735a2eb6e23b1 \ + --hash=sha256:e0c71405399ef59782600b1f0bdebc69ba12d7c9527cd268162a86273971d294 \ + --hash=sha256:e1c2ac0955f5fb87b8444316e475242d194c3f3cd0b7b6e54b889a7b6f05156f \ + --hash=sha256:e302b4e1fa856d74ff65c65888b3a37153287ce6ad5bad80b2fdf95130accec2 \ + --hash=sha256:eb67b9aa9cd69468bceb933e8e0f89fd13695746c018c4d2e6b3b84e73f3ad97 \ + --hash=sha256:edd52d667f2aa3c73233be0a821596937f24536647c12d96bfc54aa4cb04747d \ + --hash=sha256:f94d5193b2f2a9595795b83e7978b2bee1c0399da66f2f24d179c388f81fb99c \ + --hash=sha256:fa4f95a79a34afc3b5464895d091cd1911227fc3ab0441b9a37cd1817cf7db86 + # via -r ./requirements.in +protobuf==5.27.2 \ + --hash=sha256:0e341109c609749d501986b835f667c6e1e24531096cff9d34ae411595e26505 \ + --hash=sha256:176c12b1f1c880bf7a76d9f7c75822b6a2bc3db2d28baa4d300e8ce4cde7409b \ + --hash=sha256:354d84fac2b0d76062e9b3221f4abbbacdfd2a4d8af36bab0474f3a0bb30ab38 \ + --hash=sha256:4fadd8d83e1992eed0248bc50a4a6361dc31bcccc84388c54c86e530b7f58863 \ + --hash=sha256:54330f07e4949d09614707c48b06d1a22f8ffb5763c159efd5c0928326a91470 \ + --hash=sha256:610e700f02469c4a997e58e328cac6f305f649826853813177e6290416e846c6 \ + --hash=sha256:7fc3add9e6003e026da5fc9e59b131b8f22b428b991ccd53e2af8071687b4fce \ + --hash=sha256:9e8f199bf7f97bd7ecebffcae45ebf9527603549b2b562df0fbc6d4d688f14ca \ + --hash=sha256:a109916aaac42bff84702fb5187f3edadbc7c97fc2c99c5ff81dd15dcce0d1e5 \ + --hash=sha256:b848dbe1d57ed7c191dfc4ea64b8b004a3f9ece4bf4d0d80a367b76df20bf36e \ + --hash=sha256:f3ecdef226b9af856075f28227ff2c90ce3a594d092c39bee5513573f25e2714 + # via + # -r ./requirements.in + # grpcio-reflection + # grpcio-tools +quantlib==1.35 \ + --hash=sha256:00229cb7594bb9fb85c257907db969d15c3816c5d549f698c19490beb897a7c9 \ + --hash=sha256:0d1dd5801dd054652d3f7852a744813e1db9dbafa6fbaedd8472bac33ed761dc \ + --hash=sha256:196fd1f735f969c585aa78d7d0c391f36c4453002769dee96e92d87983d89417 \ + --hash=sha256:2536c0bb3a3f891b08eaf12ff0e04680ee9703316af1478b7d636a00007f7d9b \ + --hash=sha256:2a505830d516067c4d73b397a75f88cc16a580434bfaffd6e9fd6b1b99a8894b \ + --hash=sha256:2eaee3b052883b745c6e83600c553b34bdea0d4fdb50bd3ca89fb5386d5d57db \ + --hash=sha256:3a5c84a06d7c25d464e9c3079b46eaf697cdf80f203e6feea9d59c3d6ec2b91e \ + --hash=sha256:3db00a6a6d55a97ef0e43b76ac395d909a6e7f1299e345457cc5fc3ed7d1ecc1 \ + --hash=sha256:4f585d8e94da4504db74098796fd66e922dd31efebdf17ffe24b0c73418eaaca \ + --hash=sha256:5f05409bf1c6b7412bcd8809b3f7c5bc663518a69c4148a1316e138af6851290 \ + --hash=sha256:81d4f434e05591dba1918afbce0597fbfb94e2de75a4dc0a7fe434a67e43d00f \ + --hash=sha256:907d53409cdd9eaef5cfd2b3a5fa99d15ed711aeabbc3be476939df3037f7f72 \ + --hash=sha256:9c19d0e625f51d0d3afc8432946b25b519e439552cdb034a9f2de54bfad2a291 \ + --hash=sha256:a8ab1608b5389632cb188087432143659cf1b8e747cf76338dd64abcfecc2ea9 \ + --hash=sha256:a9325fbedcab7fbacc384652005d41e7cd0db1272dd82520f750c48247aaf5d8 \ + --hash=sha256:b155c02820a6981aafba306cd2fe30ff267fb9c9ad0e6efa35ff0918ba7ecd1f \ + --hash=sha256:c3ce401c8ff600b7d19d88caf74dfc0275d7703d6cb7bf3f553ffcdd6533358d \ + --hash=sha256:c7accb9bcc82581ec054e93edc6d960cc5eb8d06af1e92736b4c2eeafd393874 \ + --hash=sha256:dcea98d0bb288ae180810fff69b61ad364552ba185d12b1ca499f8ce8c74f907 \ + --hash=sha256:ddcdc2490d86869c9428023f4d1f79e5cfd8dabd29c7839cca11bf7019a923de \ + --hash=sha256:e31077867d4ba4d0030e3b36966f6fbcf9b47272040acc4d3c6138f09c966332 \ + --hash=sha256:ec249dc01cba4ab13efe02aa62765aa2238f4be32fdd65119592235d9c4c1889 + # via -r ./requirements.in +setuptools==75.1.0 \ + --hash=sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2 \ + --hash=sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538 + # via grpcio-tools diff --git a/5-appinfra/modules/htc-infra/american-option/response.proto b/5-appinfra/modules/htc-infra/american-option/response.proto new file mode 100644 index 000000000..07080cfe0 --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/response.proto @@ -0,0 +1,22 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package main; + +// Interest rate swap proto +message PricingResponse { + repeated double value = 1; +} diff --git a/5-appinfra/modules/htc-infra/american-option/service.proto b/5-appinfra/modules/htc-infra/american-option/service.proto new file mode 100644 index 000000000..163a3d4ef --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/service.proto @@ -0,0 +1,26 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package main; + +import "request.proto"; +import "response.proto"; + +service PricingService { + + // Runs the simulated library + rpc CalcPrices(PricingRequest) returns (PricingResponse) {} +} diff --git a/5-appinfra/modules/htc-infra/american-option/setup.sh b/5-appinfra/modules/htc-infra/american-option/setup.sh new file mode 100755 index 000000000..364b41063 --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/setup.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Ensure python3 is installed +if ! command -v python3 > /dev/null; then + echo "python3 needs to be installed" + exit 1 +fi + +ROOT="$(dirname "$0")" +VENV="${ROOT}/.venv" + +# Create virtual environment +[ -d "${VENV}" ] || ( + echo "creating virtual environment" + python3 -m venv "${VENV}" + "${VENV}/bin/python3" -m ensurepip + "${VENV}/bin/python3" -m pip install uv +) + +# Lock requirements if needed +if [ -f "${ROOT}/requirements.in" ] && [ "${ROOT}/requirements.in" -nt "${ROOT}/requirements.txt" ]; then + "${VENV}/bin/python3" -m uv --quiet pip compile --generate-hashes "${ROOT}/requirements.in" > "${ROOT}/requirements.txt" +fi + +# Sync +"${VENV}/bin/python3" -m uv --quiet pip sync "${ROOT}/requirements.txt" + +# Re-generate protobuf code +"${VENV}/bin/python3" -m grpc_tools.protoc --proto_path="${ROOT}" --python_out="${ROOT}" "${ROOT}"/*.proto +"${VENV}/bin/python3" -m grpc_tools.protoc -I"${ROOT}" --python_out="${ROOT}" --grpc_python_out="${ROOT}" "${ROOT}/service.proto" diff --git a/5-appinfra/modules/htc-infra/american-option/test_data.py b/5-appinfra/modules/htc-infra/american-option/test_data.py new file mode 100755 index 000000000..9af02e4f8 --- /dev/null +++ b/5-appinfra/modules/htc-infra/american-option/test_data.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# flake8: noqa + +import json +import argparse +import sys +import gzip + + +def generate_test_data(output, count: int = 1): + # Get the output file (compress if needed) + if output.name.endswith(".gz"): + output = gzip.open(output, mode="wb") + + # Generate output + for i in range(count): + output.write(bytes(generate_request(), "utf8")) + output.write(bytes("\n", "utf8")) + + +def generate_request() -> str: + pyvoldates = [ + {"year": 2021, "month": 2, "day": 8}, + {"year": 2022, "month": 2, "day": 8}, + {"year": 2023, "month": 2, "day": 8}, + {"year": 2025, "month": 2, "day": 8}, + {"year": 2027, "month": 2, "day": 8}, + ] + pyvolatilities = [ + {"nanos": 100000000}, + {"nanos": 120000000}, + {"nanos": 130000000}, + {"nanos": 150000000}, + {"nanos": 200000000}, + {"nanos": 150000000}, + {"nanos": 100000000}, + {"nanos": 200000000}, + {"nanos": 100000000}, + {"nanos": 100000000}, + {"nanos": 200000000}, + {"nanos": 100000000}, + {"nanos": 100000000}, + {"nanos": 100000000}, + {"nanos": 300000000}, + ] + goog_pyvolstrikes = [{"units": 1450}, {"units": 1500}, {"units": 1550}] + ezj_pyvolstrikes = [{"units": 570}, {"units": 590}, {"units": 610}] + py_curve_discounts = [ + {"date": {"year": 2021, "month": 2, "day": 8}, "value": {"nanos": 971974410}}, + {"date": {"year": 2022, "month": 2, "day": 8}, "value": {"nanos": 940227460}}, + {"date": {"year": 2023, "month": 2, "day": 8}, "value": {"nanos": 910740310}}, + {"date": {"year": 2025, "month": 2, "day": 8}, "value": {"nanos": 854950890}}, + {"date": {"year": 2027, "month": 2, "day": 8}, "value": {"nanos": 801367500}}, + {"date": {"year": 2030, "month": 2, "day": 8}, "value": {"nanos": 724948790}}, + {"date": {"year": 2050, "month": 2, "day": 8}, "value": {"nanos": 376020590}}, + ] + marketdata = { + "rate_curves": [ + { + "currency": "USD", + "rate_type": "RISK_FREE_CURVE", + "discounts": py_curve_discounts, + }, + { + "currency": "GBP", + "rate_type": "RISK_FREE_CURVE", + "discounts": py_curve_discounts, + }, + ], + "equity_options": [ + { + "id": "GOOG", + "currency": "USD", + "spot_price": {"units": 1500}, + "strike_dates": pyvoldates, + "strike_prices": goog_pyvolstrikes, + "implied_vols": pyvolatilities, + }, + { + "id": "EZJ", + "currency": "GBP", + "spot_price": {"units": 590}, + "strike_dates": pyvoldates, + "strike_prices": ezj_pyvolstrikes, + "implied_vols": pyvolatilities, + }, + ], + "reference_date": {"year": 2021, "month": 2, "day": 5}, + } + opt_reqs = [ + { + "short_position": True, + "expiry_date": {"year": 2022, "month": 5, "day": 21}, + "contract_amount": {"units": 10000}, + "strike": {"units": 1500}, + "equity": "GOOG", + "currency": "USD", + "business_day_convention": "MODIFIED_FOLLOWING", + "settlement_days": 2, + "is_call_option": False, + }, + { + "short_position": True, + "expiry_date": {"year": 2022, "month": 5, "day": 21}, + "contract_amount": {"units": 10000}, + "strike": {"units": 590}, + "equity": "EZJ", + "currency": "GBP", + "business_day_convention": "FOLLOWING", + "settlement_days": 2, + "is_call_option": False, + }, + { + "short_position": True, + "expiry_date": {"year": 2022, "month": 5, "day": 21}, + "contract_amount": {"units": 10000}, + "strike": {"units": 590}, + "equity": "EZJ", + "currency": "GBP", + "business_day_convention": "MODIFIED_FOLLOWING", + "settlement_days": 2, + "is_call_option": True, + }, + ] + + return json.dumps( + { + "marketdata": marketdata, + "american_option_request": opt_reqs, + } + ) + + +def main(args): + parser = argparse.ArgumentParser( + prog="generate", description="Generate test data for american-option" + ) + parser.add_argument( + "-c", + "--count", + help="Number of output records to generate", + type=int, + default=1, + ) + parser.add_argument( + "outfile", help="Output file with JSONL records", type=argparse.FileType("wb") + ) + + args = parser.parse_args() + + # Get the output file (compress if needed) + ofile = args.outfile + if args.outfile.name.endswith(".gz"): + ofile = gzip.open(ofile, mode="wb") + + # Generate output + for i in range(args.count): + ofile.write(bytes(generate_request(), "utf8")) + ofile.write(bytes("\n", "utf8")) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/5-appinfra/modules/htc-infra/apis.tf b/5-appinfra/modules/htc-infra/apis.tf new file mode 100644 index 000000000..766e8568e --- /dev/null +++ b/5-appinfra/modules/htc-infra/apis.tf @@ -0,0 +1,58 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "enabled_google_apis" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "~> 18.0" + + project_id = var.infra_project + disable_services_on_destroy = false + + activate_apis = [ + "bigquery.googleapis.com", + "parallelstore.googleapis.com", + "container.googleapis.com", + "logging.googleapis.com" + ] +} + +resource "google_project_iam_member" "team_roles_infra_project" { + for_each = toset([ + "roles/storage.objectUser", + "roles/storage.objectViewer", + "roles/pubsub.publisher", + "roles/pubsub.viewer", + "roles/pubsub.subscriber", + "roles/monitoring.viewer" + ]) + + project = var.infra_project + role = each.value + member = "principalSet://iam.googleapis.com/projects/${var.cluster_project_number}/locations/global/workloadIdentityPools/${var.cluster_project_id}.svc.id.goog/namespace/${local.namespace}" +} + +resource "google_project_iam_member" "team_roles_cluster_project" { + for_each = toset([ + "roles/storage.objectUser", + "roles/storage.objectViewer", + "roles/pubsub.publisher", + "roles/pubsub.viewer", + "roles/pubsub.subscriber", + "roles/monitoring.viewer" + ]) + + project = var.cluster_project_id + role = each.value + member = "principalSet://iam.googleapis.com/projects/${var.cluster_project_number}/locations/global/workloadIdentityPools/${var.cluster_project_id}.svc.id.goog/namespace/${local.namespace}" +} diff --git a/5-appinfra/modules/htc-infra/kubernetes/compute-classes/gpu-spot-capacity.yaml b/5-appinfra/modules/htc-infra/kubernetes/compute-classes/gpu-spot-capacity.yaml new file mode 100644 index 000000000..885190821 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/compute-classes/gpu-spot-capacity.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: cloud.google.com/v1 +kind: ComputeClass +metadata: + name: gpu-spot-capacity +spec: + autoscalingPolicy: + consolidationDelayMinutes: 2 # Mark a node as eligible for scale down after 1 minute at 10% utilisation + consolidationThreshold: 30 # Very low threshold to encourage node packing + nodePoolAutoCreation: + enabled: true + priorities: + - machineFamily: g2 + minCores: 8 + minMemoryGb: 32 + spot: true + gpu: + type: nvidia-l4 + count: 1 + - machineFamily: n1 + minCores: 8 + minMemoryGb: 32 + spot: true + gpu: + type: nvidia-tesla-t4 + count: 1 + whenUnsatisfiable: ScaleUpAnyway # Required for Autopilot + activeMigration: + optimizeRulePriority: false # Prioritize moving workloads to optimal instances diff --git a/5-appinfra/modules/htc-infra/kubernetes/compute-classes/h3-spot-capacity.yaml b/5-appinfra/modules/htc-infra/kubernetes/compute-classes/h3-spot-capacity.yaml new file mode 100644 index 000000000..c964e737a --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/compute-classes/h3-spot-capacity.yaml @@ -0,0 +1,30 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: cloud.google.com/v1 +kind: ComputeClass +metadata: + name: hpc-h3-spot-capacity +spec: + autoscalingPolicy: + consolidationDelayMinutes: 2 # Mark a node as eligible for scale down after 1 minute at 10% utilisation + consolidationThreshold: 30 # Very low threshold to encourage node packing + nodePoolAutoCreation: + enabled: true + priorities: + - machineType: "h3-standard-88" + spot: true + whenUnsatisfiable: ScaleUpAnyway # Required for Autopilot + activeMigration: + optimizeRulePriority: false # Prioritize moving workloads to optimal instances diff --git a/5-appinfra/modules/htc-infra/kubernetes/compute-classes/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/compute-classes/kustomization.yaml new file mode 100644 index 000000000..ae63564fa --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/compute-classes/kustomization.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- gpu-spot-capacity.yaml +- h3-spot-capacity.yaml +- n-spot-capacity.yaml diff --git a/5-appinfra/modules/htc-infra/kubernetes/compute-classes/n-spot-capacity.yaml b/5-appinfra/modules/htc-infra/kubernetes/compute-classes/n-spot-capacity.yaml new file mode 100644 index 000000000..ac70c91bb --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/compute-classes/n-spot-capacity.yaml @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: cloud.google.com/v1 +kind: ComputeClass +metadata: + name: spot-capacity +spec: + autoscalingPolicy: + consolidationDelayMinutes: 2 + consolidationThreshold: 70 + nodePoolAutoCreation: + enabled: true + priorities: + - nodepools: [spot-nodes-1] + - machineFamily: "n2" + minCores: 4 + minMemoryGb: 16 + spot: true + - machineFamily: "e2" + minCores: 4 + minMemoryGb: 16 + spot: true + - machineFamily: "n1" + minCores: 4 + minMemoryGb: 16 + spot: true + - machineFamily: "n4" + minCores: 4 + minMemoryGb: 16 + spot: true + whenUnsatisfiable: ScaleUpAnyway + activeMigration: + optimizeRulePriority: false diff --git a/5-appinfra/modules/htc-infra/kubernetes/kueue/README.md b/5-appinfra/modules/htc-infra/kubernetes/kueue/README.md new file mode 100644 index 000000000..c057dc250 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/kueue/README.md @@ -0,0 +1,68 @@ +# Installing Kueue +This document provides a basic guide for using Kustomize to deploy and manage Kueue installations. + +It includes integration with Google Cloud Monitoring and Google Cloud Managed Service for Prometheus and a includes: +- A [custom-configured released version](https://kueue.sigs.k8s.io/docs/installation/#install-a-custom-configured-released-version) to enable `enableClusterQueueResources` +- Enabled [visibility API](https://kueue.sigs.k8s.io/docs/installation/#add-visibility-api-to-monitor-pending-workloads) + +**Prerequisites:** + +- Kustomize: Ensure you have Kustomize installed on your system. You can download it from the official Kustomize releases page. +- kubectl: You'll need kubectl to interact with your Kubernetes cluster. + +## Deploy Kueue with GMP Configuration and Cloud Monitoring Dashboard +**Deploy Kueue** + +Deploy for GKE Standard +```bash +kustomize build overlays/gke-standard/ | kubectl apply --server-side -f - +``` + +Deploy for GKE Autopilot +```bash +kustomize build overlays/gke-autopilot/ | kubectl apply --server-side -f - +``` + +**Deploy Monitoring Dashboard** + +```bash +gcloud monitoring dashboards create --project=$PROJECT_ID --config-from-file=dashboard/kueue-dashboard.json +``` + +## Querying Metrics + +You can also query Kueue metrics directly using the [Google Cloud Monitoring - Metrics explorer](https://console.cloud.google.com/monitoring/metrics-explorer) interface. Both PromQL and MQL are supported for querying. + +For more information, refer to the [Cloud Monitoring Documentation](https://cloud.google.com/monitoring/charts/metrics-explorer). + +### Example Queries + +Here are some sample PromQL queries to help you get started with monitoring your Kueue system: + +#### Job Throughput + +```promql +sum(rate(kueue_admitted_workloads_total[5m])) by (cluster_queue) +``` + +This query calculates the per-second rate of admitted workloads over 5 minutes for each cluster queue. Summing them provides the overall system throughput, while breaking it down by queue helps pinpoint potential bottlenecks. + +#### Resource Utilization (`requires metrics.enableClusterQueueResources`) + +```promql +sum(kueue_cluster_queue_resource_usage{resource="cpu"}) by (cluster_queue) / sum(kueue_cluster_queue_nominal_quota{resource="cpu"}) by (cluster_queue) +``` + +This query calculates the ratio of current CPU usage to the nominal CPU quota for each queue. A value close to 1 indicates high CPU utilization. You can adapt this for memory or other resources by changing the resource label. + +>__Important__: This query requires the metrics.enableClusterQueueResources setting to be enabled in your Kueue manager's configuration. To enable this setting, follow the instructions in the Kueue installation documentation: [https://kueue.sigs.k8s.io/docs/installation/#install-a-custom-configured-released-version](https://kueue.sigs.k8s.io/docs/installation/#install-a-custom-configured-released-version) +#### Queue Wait Times +```promql +histogram_quantile(0.9, kueue_admission_wait_time_seconds_bucket{cluster_queue="QUEUE_NAME"}) +``` +This query provides the 90th percentile wait time for workloads in a specific queue. You can modify the quantile value (e.g., 0.5 for median, 0.99 for 99th percentile) to understand the wait time distribution. Replace `QUEUE_NAME` with the actual name of the queue you want to monitor. + +## Preview of Dashboard + +![Dashboard 1](../../images/kueue_cloud_monitoring_1.png) +![Dashboard 2](../../images/kueue_cloud_monitoring_2.png) diff --git a/5-appinfra/modules/htc-infra/kubernetes/kueue/base/apiserver-reader.yaml b/5-appinfra/modules/htc-infra/kubernetes/kueue/base/apiserver-reader.yaml new file mode 100644 index 000000000..59d73d294 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/kueue/base/apiserver-reader.yaml @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kueue-apiserver-reader + namespace: kube-system +roleRef: + kind: Role + name: extension-apiserver-authentication-reader + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system diff --git a/5-appinfra/modules/htc-infra/kubernetes/kueue/base/kueue-manager-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/kueue/base/kueue-manager-config.yaml new file mode 100644 index 000000000..836559b08 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/kueue/base/kueue-manager-config.yaml @@ -0,0 +1,62 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ConfigMap +apiVersion: v1 +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-config + namespace: kueue-system +data: + controller_manager_config.yaml: | + apiVersion: config.kueue.x-k8s.io/v1beta1 + kind: Configuration + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: :8080 + enableClusterQueueResources: true + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io + controller: + groupKindConcurrency: + Job.batch: 5 + Pod: 5 + Workload.kueue.x-k8s.io: 5 + LocalQueue.kueue.x-k8s.io: 1 + ClusterQueue.kueue.x-k8s.io: 1 + ResourceFlavor.kueue.x-k8s.io: 1 + clientConnection: + qps: 50 + burst: 100 + integrations: + frameworks: + - "pod" + - "deployment" + - "statefulset" + - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" diff --git a/5-appinfra/modules/htc-infra/kubernetes/kueue/base/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/kueue/base/kustomization.yaml new file mode 100644 index 000000000..b00e40a89 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/kueue/base/kustomization.yaml @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- promethus-gmp.yaml +- apiserver-reader.yaml +- https://github.com/kubernetes-sigs/kueue/releases/latest/download/manifests.yaml +# - https://github.com/kubernetes-sigs/kueue/releases/latest/download/visibility-apf.yaml +patches: +- patch: |- + - op: replace + path: /subjects/0/namespace + value: "gmp-system" # Default value in the base + target: + kind: RoleBinding + name: gmp-system:collector:kueue-secret-reader + version: v1 +- path: kueue-manager-config.yaml +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: not-important-ignore + namespace: kueue-system + unsetOnly: true diff --git a/5-appinfra/modules/htc-infra/kubernetes/kueue/base/promethus-gmp.yaml b/5-appinfra/modules/htc-infra/kubernetes/kueue/base/promethus-gmp.yaml new file mode 100644 index 000000000..a936645af --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/kueue/base/promethus-gmp.yaml @@ -0,0 +1,84 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kueue-metrics-reader +--- +apiVersion: v1 +kind: Secret +metadata: + name: kueue-metrics-reader-token + annotations: + kubernetes.io/service-account.name: kueue-metrics-reader +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kueue-secret-reader +rules: +- resources: + - secrets + apiGroups: [""] + verbs: ["get", "list", "watch"] + resourceNames: ["kueue-metrics-reader-token"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gmp-system:collector:kueue-secret-reader +roleRef: + name: kueue-secret-reader + kind: Role + apiGroup: rbac.authorization.k8s.io +subjects: +- name: collector + namespace: gmp-system + kind: ServiceAccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kueue-metrics-reader +subjects: +- kind: ServiceAccount + name: kueue-metrics-reader +roleRef: + kind: ClusterRole + name: kueue-metrics-reader + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: monitoring.googleapis.com/v1 +kind: PodMonitoring +metadata: + name: kueue +spec: + selector: + matchLabels: + control-plane: controller-manager + endpoints: + - port: https + interval: 30s + path: /metrics + scheme: https + tls: + insecureSkipVerify: true + authorization: + type: Bearer + credentials: + secret: + name: kueue-metrics-reader-token + key: token diff --git a/5-appinfra/modules/htc-infra/kubernetes/kueue/dashboard/kueue-dashboard.json b/5-appinfra/modules/htc-infra/kubernetes/kueue/dashboard/kueue-dashboard.json new file mode 100644 index 000000000..0148c2f5e --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/kueue/dashboard/kueue-dashboard.json @@ -0,0 +1,700 @@ +{ + "displayName": "Kueue - Queue and Cluster", + "mosaicLayout": { + "columns": 48, + "tiles": [ + { + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/kueue_admission_attempts_total/counter\" resource.type=\"prometheus_target\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_RATE", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [] + } + }, + "unitOverride": "", + "outputFullDuration": false + }, + "sparkChartView": { + "sparkChartType": "SPARK_LINE" + }, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Admissions /s", + "id": "" + } + }, + { + "xPos": 8, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(histogram_quantile(0.99, rate(kueue_admission_attempt_duration_seconds_bucket[60s]))>0) OR on() vector(0) ", + "unitOverride": "", + "outputFullDuration": false + }, + "sparkChartView": { + "sparkChartType": "SPARK_LINE" + }, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Admission Latency [99% percentile]", + "id": "" + } + }, + { + "xPos": 16, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(kueue_admitted_active_workloads)", + "unitOverride": "", + "outputFullDuration": true + }, + "sparkChartView": { + "sparkChartType": "SPARK_LINE" + }, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Active Workloads", + "id": "" + } + }, + { + "xPos": 24, + "width": 12, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(kueue_pending_workloads)", + "unitOverride": "", + "outputFullDuration": true + }, + "sparkChartView": { + "sparkChartType": "SPARK_LINE" + }, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "The total number of all pending workloads in all cluster queues", + "id": "" + } + }, + { + "xPos": 36, + "width": 12, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "count(rate(kubernetes_io:node_cpu_core_usage_time{monitored_resource=\"k8s_node\",${cluster_name}, ${location}, ${project_id}}[${__interval}]))", + "unitOverride": "", + "outputFullDuration": false + }, + "sparkChartView": { + "sparkChartType": "SPARK_LINE" + }, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "The total number of nodes in a cluster", + "id": "" + } + }, + { + "xPos": 1, + "yPos": 8, + "width": 22, + "height": 16, + "widget": { + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "prometheusQuery": "sum by(cluster_queue)(kueue_admitted_active_workloads)", + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "title": "Active workloads per cluster queue", + "id": "" + } + }, + { + "xPos": 23, + "yPos": 8, + "width": 24, + "height": 16, + "widget": { + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "prometheusQuery": "sum by(cluster_queue)(kueue_pending_workloads)", + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "title": "Pending workloads per cluster queue", + "id": "" + } + }, + { + "yPos": 8, + "width": 48, + "height": 32, + "widget": { + "title": "Cluster Queues", + "collapsibleGroup": { + "collapsed": false + }, + "id": "" + } + }, + { + "yPos": 40, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(rate(kubernetes_io:node_cpu_core_usage_time{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}])) \n/ \nsum(avg_over_time(kubernetes_io:node_cpu_total_cores{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}])) * 100", + "unitOverride": "%", + "outputFullDuration": false + }, + "gaugeView": { + "lowerBound": 0, + "upperBound": 100 + }, + "thresholds": [ + { + "label": "", + "value": 95, + "color": "RED", + "direction": "ABOVE", + "targetAxis": "TARGET_AXIS_UNSPECIFIED" + }, + { + "label": "", + "value": 90, + "color": "YELLOW", + "direction": "ABOVE", + "targetAxis": "TARGET_AXIS_UNSPECIFIED" + } + ], + "dimensions": [], + "measures": [] + }, + "title": "CPU Utilization", + "id": "" + } + }, + { + "xPos": 8, + "yPos": 40, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(rate(kubernetes_io:container_cpu_core_usage_time{monitored_resource=\"k8s_container\", ${project_id}, ${cluster_name}, ${location}}[${__interval}]))\n/\nsum(avg_over_time(kubernetes_io:container_cpu_request_cores{monitored_resource=\"k8s_container\", ${project_id}, ${cluster_name}, ${location}}[${__interval}])) * 100", + "unitOverride": "%", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "CPU Request Utilization", + "id": "" + } + }, + { + "xPos": 16, + "yPos": 40, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(rate(kubernetes_io:container_cpu_core_usage_time{monitored_resource=\"k8s_container\", ${project_id}, ${cluster_name}, ${location}}[${__interval}]))\n/\nsum(avg_over_time(kubernetes_io:container_cpu_limit_cores{monitored_resource=\"k8s_container\", ${project_id}, ${cluster_name}, ${location}}[${__interval}])) * 100", + "unitOverride": "%", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "CPU Limit Utilization", + "id": "" + } + }, + { + "xPos": 24, + "yPos": 40, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(avg_over_time(kubernetes_io:node_memory_used_bytes{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}])) \n/\nsum(avg_over_time(kubernetes_io:node_memory_total_bytes{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}])) * 100\n", + "unitOverride": "%", + "outputFullDuration": false + }, + "gaugeView": { + "lowerBound": 0, + "upperBound": 100 + }, + "thresholds": [ + { + "label": "", + "value": 95, + "color": "RED", + "direction": "ABOVE", + "targetAxis": "TARGET_AXIS_UNSPECIFIED" + }, + { + "label": "", + "value": 90, + "color": "YELLOW", + "direction": "ABOVE", + "targetAxis": "TARGET_AXIS_UNSPECIFIED" + } + ], + "dimensions": [], + "measures": [] + }, + "title": "Memory Utilization", + "id": "" + } + }, + { + "xPos": 32, + "yPos": 40, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(avg_over_time(kubernetes_io:container_memory_used_bytes{monitored_resource=\"k8s_container\", ${project_id}, ${cluster_name}, ${location}}[${__interval}]))\n/\nsum(avg_over_time(kubernetes_io:container_memory_request_bytes{monitored_resource=\"k8s_container\", ${project_id}, ${cluster_name}, ${location}}[${__interval}])) * 100", + "unitOverride": "%", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Memory Request Utilization", + "id": "" + } + }, + { + "xPos": 40, + "yPos": 40, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(avg_over_time(kubernetes_io:container_memory_used_bytes{monitored_resource=\"k8s_container\", ${project_id}, ${cluster_name}, ${location}}[${__interval}]))\n/\nsum(avg_over_time(kubernetes_io:container_memory_limit_bytes{monitored_resource=\"k8s_container\", ${project_id}, ${cluster_name}, ${location}}[${__interval}])) * 100", + "unitOverride": "%", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Memory Limit Utilization", + "id": "" + } + }, + { + "yPos": 48, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "count(rate(kubernetes_io:node_cpu_core_usage_time{monitored_resource=\"k8s_node\",${cluster_name}, ${location}, ${project_id}}[${__interval}]))", + "unitOverride": "", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Total Nodes", + "id": "" + } + }, + { + "xPos": 8, + "yPos": 48, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(avg_over_time(kubernetes_io:node_cpu_total_cores{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}]))", + "unitOverride": "", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Total Cores", + "id": "" + } + }, + { + "xPos": 16, + "yPos": 48, + "width": 8, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(avg_over_time(kubernetes_io:node_cpu_allocatable_cores{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}]))", + "unitOverride": "", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Allocatable Cores", + "id": "" + } + }, + { + "xPos": 24, + "yPos": 48, + "width": 12, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(avg_over_time(kubernetes_io:node_memory_total_bytes{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}]))", + "unitOverride": "", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Total Memory", + "id": "" + } + }, + { + "xPos": 36, + "yPos": 48, + "width": 12, + "height": 8, + "widget": { + "scorecard": { + "timeSeriesQuery": { + "prometheusQuery": "sum(avg_over_time(kubernetes_io:node_memory_allocatable_bytes{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}]))", + "unitOverride": "", + "outputFullDuration": false + }, + "blankView": {}, + "thresholds": [], + "dimensions": [], + "measures": [] + }, + "title": "Allocatable Memory", + "id": "" + } + }, + { + "yPos": 56, + "width": 24, + "height": 16, + "widget": { + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "prometheusQuery": "count(rate(kubernetes_io:node_cpu_core_usage_time{monitored_resource=\"k8s_node\",${cluster_name}, ${location}, ${project_id}}[${__interval}]))", + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "STACKED_AREA", + "legendTemplate": "", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "title": "Node Count", + "id": "" + } + }, + { + "xPos": 24, + "yPos": 56, + "width": 24, + "height": 16, + "widget": { + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "prometheusQuery": "sum(rate(kubernetes_io:node_cpu_core_usage_time{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}])) / sum(avg_over_time(kubernetes_io:node_cpu_total_cores{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}])) * 100", + "unitOverride": "%", + "outputFullDuration": false + }, + "plotType": "STACKED_AREA", + "legendTemplate": "", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "prometheusQuery": "sum(avg_over_time(kubernetes_io:node_memory_used_bytes{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}])) / sum(avg_over_time(kubernetes_io:node_memory_total_bytes{monitored_resource=\"k8s_node\", ${cluster_name}, ${location}, ${project_id}}[${__interval}])) * 100", + "unitOverride": "%", + "outputFullDuration": false + }, + "plotType": "STACKED_AREA", + "legendTemplate": "", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "title": "Node CPU & Memory Utilization ", + "id": "" + } + }, + { + "xPos": 1, + "yPos": 24, + "width": 22, + "height": 16, + "widget": { + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "prometheusQuery": "sum(kueue_cluster_queue_resource_usage{resource=\"cpu\"}) by (cluster_queue) / sum(kueue_cluster_queue_nominal_quota{resource=\"cpu\"}) by (cluster_queue)", + "unitOverride": "1", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [ + { + "label": "", + "value": 1, + "color": "COLOR_UNSPECIFIED", + "direction": "DIRECTION_UNSPECIFIED", + "targetAxis": "Y1" + } + ], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "title": "Cluster queue quota utilization", + "id": "" + } + }, + { + "xPos": 23, + "yPos": 24, + "width": 24, + "height": 16, + "widget": { + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "prometheusQuery": "sum(kueue_admitted_active_workloads)", + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + }, + { + "timeSeriesQuery": { + "prometheusQuery": "sum(kueue_pending_workloads)", + "unitOverride": "", + "outputFullDuration": false + }, + "plotType": "LINE", + "legendTemplate": "", + "targetAxis": "Y1", + "dimensions": [], + "measures": [], + "breakdowns": [] + } + ], + "thresholds": [], + "yAxis": { + "label": "", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR", + "showLegend": false, + "displayHorizontal": false + } + }, + "title": "Total Admitted & Pending Workloads", + "id": "" + } + }, + { + "yPos": 40, + "width": 48, + "height": 32, + "widget": { + "title": "Cluster GKE Nodes", + "collapsibleGroup": { + "collapsed": false + }, + "id": "" + } + } + ] + }, + "dashboardFilters": [ + { + "labelKey": "project_id", + "templateVariable": "project_id", + "stringValue": "", + "filterType": "RESOURCE_LABEL", + "valueType": "STRING" + }, + { + "labelKey": "location", + "templateVariable": "location", + "stringValue": "", + "filterType": "RESOURCE_LABEL", + "valueType": "STRING" + }, + { + "labelKey": "cluster_name", + "templateVariable": "cluster_name", + "stringValue": "", + "filterType": "RESOURCE_LABEL", + "valueType": "STRING" + }, + { + "labelKey": "namespace", + "templateVariable": "namespace", + "stringValue": "", + "filterType": "RESOURCE_LABEL", + "valueType": "STRING" + } + ], + "labels": {} +} diff --git a/5-appinfra/modules/htc-infra/kubernetes/kueue/overlays/gke-autopilot/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/kueue/overlays/gke-autopilot/kustomization.yaml new file mode 100644 index 000000000..012bd4953 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/kueue/overlays/gke-autopilot/kustomization.yaml @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- patch: |- + - op: replace + path: /subjects/0/namespace + value: "gke-gmp-system" + target: + kind: RoleBinding + name: gmp-system:collector:kueue-secret-reader + version: v1 +resources: +- ../../base diff --git a/5-appinfra/modules/htc-infra/kubernetes/kueue/overlays/gke-standard/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/kueue/overlays/gke-standard/kustomization.yaml new file mode 100644 index 000000000..3f6a5964d --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/kueue/overlays/gke-standard/kustomization.yaml @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- patch: |- + - op: replace + path: /subjects/0/namespace + value: "gmp-system" + target: + kind: RoleBinding + name: gmp-system:collector:kueue-secret-reader + version: v1 +resources: +- ../../base diff --git a/5-appinfra/modules/htc-infra/kubernetes/placeholder-pods.yaml b/5-appinfra/modules/htc-infra/kubernetes/placeholder-pods.yaml new file mode 100644 index 000000000..03ba55dc2 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/placeholder-pods.yaml @@ -0,0 +1,50 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: balloon-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: balloon-pod + template: + metadata: + labels: + app: balloon-pod + spec: + priorityClassName: placeholder-priority + nodeSelector: + cloud.google.com/compute-class: spot-capacity + containers: + - name: balloon-container + image: busybox + command: ["sleep", "infinity"] + resources: + requests: + cpu: "1" + memory: "128Mi" + limits: + cpu: "1" + memory: "128Mi" + volumeMounts: + - mountPath: /data + name: parallelstore-volume + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc-us-east4-b + # claimName: parallelstore-pvc-us-central1-b diff --git a/5-appinfra/modules/htc-infra/kubernetes/priority-classes/classes.yaml b/5-appinfra/modules/htc-infra/kubernetes/priority-classes/classes.yaml new file mode 100644 index 000000000..9e78c5414 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/priority-classes/classes.yaml @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: placeholder-priority +value: -10 +preemptionPolicy: Never +globalDefault: false +description: "Placeholder Pod priority." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: default-priority +value: 0 +preemptionPolicy: PreemptLowerPriority +globalDefault: true +description: "The global default priority. Will preempt placeholder Pods." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: higher-priority +value: 10 +preemptionPolicy: PreemptLowerPriority +globalDefault: false +description: "Higher Priority, will preempt placeholder Pods." diff --git a/5-appinfra/modules/htc-infra/kubernetes/priority-classes/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/priority-classes/kustomization.yaml new file mode 100644 index 000000000..327c8d5b3 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/priority-classes/kustomization.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- classes.yaml diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/.dockerignore b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/.dockerignore new file mode 100644 index 000000000..5733231b4 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/.dockerignore @@ -0,0 +1,3 @@ +**/k8s/* +**/k8s +README.md diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/Dockerfile b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/Dockerfile new file mode 100644 index 000000000..ab72e5359 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/Dockerfile @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:24.04 + +# Install FIO and supporting tools +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + fio \ + jq \ + python3 \ + python3-pip \ + tini \ + && rm -rf /var/lib/apt/lists/* + +# Create directories for tests and results +RUN mkdir -p /fio/jobs /fio/results + +ENTRYPOINT ["/usr/bin/tini", "--"] + +CMD ["sh", "-c", "echo Using config file: $CONFIG_PATH; fio --directory=$MOUNT_PATH --output-format=json+ $CONFIG_PATH | jq -c"] diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/README.md b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/README.md new file mode 100644 index 000000000..7168a3a34 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/README.md @@ -0,0 +1,152 @@ +# FIO Benchmark for Google Cloud Storage (GCS) and Parallelstore + +This repository provides an example Docker image and Kubernetes configurations for using FIO (Flexible I/O Tester) to benchmark Google Cloud Storage (GCS) buckets and Parallelstore. It leverages Kubernetes Jobs to run FIO tests and provides separate configurations for GCS and Parallelstore. + +## Overview + +The setup includes: + +* **Dockerfile:** Builds a Docker image containing FIO, `jq`, Python 3, and necessary tools. +* **entrypoint.sh:** A script that handles FIO execution, mount point checks, configuration validation, and cleanup. +* **k8s:** Directory containing sample GCS and Parallelstore configs for testing + * **gcs:** Kubernetes Job definition for running FIO tests against a GCS bucket mounted using the GCSFuse CSI driver. + * **parallelstore:** Kubernetes Job definition for running FIO tests against a Parallelstore Persistent Volume. + +## Getting Started + +### Prerequisites + +* Docker installed. +* kubectl configured to connect to your Kubernetes cluster. +* A GCS bucket for testing +* A Parallelstore instance for testing +* A Docker registry (e.g., Google Container Registry) to push the built Docker image. + + ```bash + export PROJECT_ID=YOUR_PROJECT_ID + export REGISTRY=YOUR_REGISTRY_NAME + ``` + +### Building and Deploying + +1. **Build the Docker image:** + + ```bash + docker build -t us-docker.pkg.dev/$PROJECT_ID/$REGISTRY/fio:latest . + ``` + + Replace `us-docker.pkg.dev/PROJECT_ID/REGISTRY/fio:latest` with your desired image name and tag. + +2. **Push the Docker image to your registry:** + + ```bash + docker push us-docker.pkg.dev/$PROJECT_ID/$REGISTRY/fio:latest + ``` +3. **Add your variables:** + + - In both directory there is a `fio-config.yaml` file which is a config map that configures FIO. Modify this to set the behaviour of your test. + + - Modify the `kustomization.yaml` file and set the variables in the configMapGenerator. + + +4. **Deploy the Config and Job:** + + ```bash + kustomize build k8s/parallelstore | kubectl apply -f - + ``` + + ```bash + kustomize build k8s/gcs | kubectl apply -f - + ``` + + +### Monitoring and Results + +* **Check job status:** + + ```bash + kubectl get jobs -l app=fio-test + ``` + +* **View job logs:** + + ```bash + kubectl logs -l app=fio-test -f + ``` + + This will stream the FIO output, which is formatted as JSON. You can use `jq` to parse and analyze the results. For example: + + ```bash + kubectl logs -l app=fio-test -f | jq . + ``` + +* **Cleanup:** + + The Kubernetes Jobs are configured with `ttlSecondsAfterFinished` to automatically clean up after completion. You can also manually delete the jobs: + + ```bash + kubectl delete job -l app=fio-test + ``` + +## Configuration Details + +### fio-config.yaml + +* `[global]`: Defines global FIO parameters. `filename_format` uses the pod name to ensure unique filenames for each run. +* `[job1]`: Defines the I/O workload. Adjust parameters like `rw`, `rwmixread`, `blocksize`, `filesize`, etc., to suit your testing needs. +* `[cleanup]`: Defines a cleanup job that trims (deletes) the files created during the test. + +### Parallelstore Options: + +* `STORAGECLASS`: The storage class name for your Parallelstore instance (without the project/location prefix). +* `PROJECT_ID`: Your Google Cloud project ID. +* `LOCATION`: The region and zone of your Parallelstore instance (e.g., `us-central1-b`). +* `INSTANCE_NAME`: The name of your Parallelstore instance. +* `STORAGE_SIZE`: The size of the PersistentVolume to create (e.g., `21000Gi`). +* `ACCESS_POINTS`: Comma-separated list of access points for the Parallelstore instance. +* `NETWORK`: The VPC network name of your Parallelstore instance. +* `MOUNT_LOCALITY`: The mount locality for the Parallelstore instance. +* `COMPLETIONS`: The number of FIO job completions. +* `PARALLELISM`: The number of parallel FIO jobs. + +## GCS Options: + +* `BUCKETNAME`: The name of your GCS bucket. +* `COMPLETIONS`: The number of FIO job completions. +* `PARALLELISM`: The number of parallel FIO jobs. + + +## Analyzing Results in BigQuery + +The FIO results are logged as JSON and can be efficiently queried in BigQuery. Ensure that your Kubernetes logs are being exported to BigQuery. You can then use a query similar to the following to analyze the results: + +```sql +SELECT + -- Timestamp and identifiers + timestamp, + JSON_EXTRACT_SCALAR(resource.labels, '$.pod_name') as pod_name, + -- Throughput and IOPS + CAST(JSON_EXTRACT_SCALAR(json_payload, '$.jobs[0].read.bw') AS FLOAT64) AS throughput_kbs, + CAST(JSON_EXTRACT_SCALAR(json_payload, '$.jobs[0].read.iops') AS FLOAT64) AS iops, + -- Data read + CAST(JSON_EXTRACT_SCALAR(json_payload, '$.jobs[0].read.io_bytes') AS FLOAT64)/1048576 AS data_read_mb, + -- Latency percentiles (in milliseconds) + CAST(JSON_EXTRACT_SCALAR(json_payload, "$.jobs[0].read.clat_ns.percentile['50.000000']") AS FLOAT64)/1000000 AS median_latency_ms, + CAST(JSON_EXTRACT_SCALAR(json_payload, "$.jobs[0].read.clat_ns.percentile['90.000000']") AS FLOAT64)/1000000 AS p90_latency_ms, + CAST(JSON_EXTRACT_SCALAR(json_payload, "$.jobs[0].read.clat_ns.percentile['99.000000']") AS FLOAT64)/1000000 AS p99_latency_ms, + -- Runtime + CAST(JSON_EXTRACT_SCALAR(json_payload, '$.jobs[0].job_runtime') AS FLOAT64)/1000 AS runtime_seconds +FROM + `PROJECT_ID.all_logging_bq_link._Default` +WHERE + resource.type = "k8s_container" + AND JSON_EXTRACT_SCALAR(resource.labels, '$.container_name') = 'fio' + AND JSON_EXTRACT_SCALAR(resource.labels, '$.pod_name') like 'fio-test-%' + AND json_payload IS NOT NULL +ORDER BY timestamp DESC; +``` + + +## Note: + +This example provides a starting point for benchmarking GCS and Parallelstore using FIO. You can customize the configurations to meet your specific testing requirements. Remember to consult the FIO documentation for more advanced configuration options. diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/entrypoint.sh b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/entrypoint.sh new file mode 100644 index 000000000..1258875c9 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/entrypoint.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for environment variables +MOUNT_PATH=${MOUNT_PATH:-"/data"} +CONFIG_PATH=${CONFIG_PATH:-"/etc/fio/fio.conf"} + +# Function to handle SIGTERM and SIGINT +term_handler() { + echo "Signal received. Stopping FIO gracefully..." + + # Find FIO processes that are children of *this* script. + fio_pid=$(pgrep -P $$ fio) + + if [ -n "$fio_pid" ]; then + echo "Sending SIGINT to FIO process(es): $fio_pid" + kill -INT "$fio_pid" + else + echo "FIO process not found." + fi + + # Wait for FIO to exit (with a timeout). + wait_counter=0 + while [ -n "$(pgrep -P $$ fio)" ] && [ "$wait_counter" -lt 10 ]; do + echo "Waiting for FIO to exit..." + sleep 1 + wait_counter=$((wait_counter + 1)) + done + + if [ -n "$(pgrep -P $$ fio)" ]; then + echo "FIO did not exit gracefully after 10 seconds. Killing forcefully." + pkill -9 fio || echo "pkill -9 fio returned non-zero (likely no FIO process running)" + fi + + + echo "Exiting entrypoint script." + exit 0 # Exit with success code (important!) +} + +# Trap SIGTERM and SIGINT. SIGKILL cannot be trapped. +trap term_handler SIGTERM SIGINT + +# Function to check if path is mounted +check_mount() { + local path=$1 + local max_attempts=30 + local attempt=1 + + echo "Checking mount point: $path" + + while ! mountpoint -q "${path}"; do + if [ $attempt -ge $max_attempts ]; then + echo "Mount point ${path} not ready after ${max_attempts} attempts. Exiting." + exit 1 + fi + echo "Waiting for ${path} to be mounted... (attempt $attempt/$max_attempts)" + sleep 5 + attempt=$((attempt + 1)) + done + + echo "Mount point ${path} is ready" +} + +# Function to validate FIO config +validate_fio_config() { + if [ ! -f "${CONFIG_PATH}" ]; then + echo "Error: FIO config file ${CONFIG_PATH} not found" + exit 1 + fi +} + +# Main execution +echo "Starting FIO test runner" +echo "Mount path: ${MOUNT_PATH}" +echo "Config file: ${CONFIG_PATH}" + +# Run checks +check_mount "${MOUNT_PATH}" +validate_fio_config + +# Run FIO in the background, capturing output to a variable +echo "Starting FIO benchmark..." +output=$(fio --directory="${MOUNT_PATH}" \ + --output-format=json+ \ + "${CONFIG_PATH}") +fio_pid=$! + +# Wait for the FIO process +wait "$fio_pid" + +# Pipe the captured output to jq +echo "$output" | jq -c + +# Check FIO exit status +FIO_STATUS=$? + +#If termination due to signal, make sure to exit clean +if [ $FIO_STATUS -eq 130 ] || [ $FIO_STATUS -eq 128 ] || [ $FIO_STATUS -eq 0 ]; then + echo "FIO terminated cleanly" + exit 0 +fi + + +if [ $FIO_STATUS -ne 0 ]; then + echo "FIO failed with exit status: ${FIO_STATUS}" + exit $FIO_STATUS +fi + +echo "FIO benchmark completed successfully" diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/cleanup-disk.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/cleanup-disk.yaml new file mode 100644 index 000000000..a28da8d05 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/cleanup-disk.yaml @@ -0,0 +1,57 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: data-cleanup-job +spec: + completions: 1 + parallelism: 1 + template: + spec: + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: higher-priority + containers: + - name: cleanup + image: busybox + command: ["sh", "-c"] + env: + - name: OUTPUT_DIR + value: test_files + args: + - | + echo "Data cleanup starting..." + if [ -z "${OUTPUT_DIR}" ]; then + echo "OUTPUT_DIR not set, skipping deletion." + else + echo "Deleting contents of: /data/${OUTPUT_DIR}" + rm -rf /data/${OUTPUT_DIR} + fi + file_count=$(find /data/${OUTPUT_DIR} -mindepth 1 -print0 | wc -l --files0-from=-) + if [ "$file_count" -gt 0 ]; then + echo "Error: Files remaining after cleanup. Count: $file_count" + exit 1 + fi + echo "Data cleanup complete." + volumeMounts: + - name: parallelstore-volume + mountPath: /data + restartPolicy: Never + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc + ttlSecondsAfterFinished: 120 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/configmap.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/configmap.yaml new file mode 100644 index 000000000..4061968eb --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/configmap.yaml @@ -0,0 +1,116 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: file-generator-config +data: + generate_files.sh: | + #!/bin/bash + + RETRY_COUNTER=0 + MAX_RETRIES=5 + SLEEP_TIME=1 + + # Check if OUTPUT_DIR environment variable is set, otherwise use default + if [ -z "${OUTPUT_DIR}" ]; then + OUTPUT_DIR="/data" # Default to /data inside the container + echo "OUTPUT_DIR not set, using default: ${OUTPUT_DIR}" + fi + + echo "Loading Data in to $OUTPUT_DIR" + echo "Waiting for Parallelstore mount..." + + OUTPUT_BASE_DIR=$(dirname "${OUTPUT_DIR}") + + while [ $RETRY_COUNTER -lt $MAX_RETRIES ]; do + if mountpoint -q $OUTPUT_BASE_DIR; then + echo "[$(date +%Y-%m-%d_%H:%M:%S)] Parallelstore mount detected at $OUTPUT_BASE_DIR" + break + fi + echo "[$(date +%Y-%m-%d_%H:%M:%S)] Parallelstore mount not detected, retrying in $SLEEP_TIME seconds..." + sleep $SLEEP_TIME + COUNTER=$((COUNTER+1)) + SLEEP_TIME=$((SLEEP_TIME * 2)) # Exponential backoff + if [ $RETRY_COUNTER -eq $MAX_RETRIES ]; then + echo "[$(date +%Y-%m-%d_%H:%M:%S)] Failed to detect Parallelstore mount at $OUTPUT_BASE_DIR after multiple retries" + # Call cleanup function if needed + cleanup # Make sure the 'cleanup' function is defined if you use it. + exit 1 + fi + done + + + # Create or clean directory for test files (using /data) + if [ -d "${OUTPUT_DIR}" ]; then + echo "Cleaning existing directory: ${OUTPUT_DIR}" + rm -rf "${OUTPUT_DIR}"/* + else + echo "Creating directory: ${OUTPUT_DIR}" + fi + mkdir -p "${OUTPUT_DIR}" + + # Function to generate a single file using FIO + generate_file() { + file_num=$1 + size=$2 + size_arg="${size_kb}K" + fio --name=generate_file_${file_num} \ + --ioengine=libaio \ + --rw=write \ + --bs=4k \ + --direct=1 \ + --size=${size_arg} \ + --filename="${OUTPUT_DIR}/testfile_${file_num}" \ + --thread \ + --group_reporting \ + --minimal + # >/dev/null 2>&1 # Redirect output to /dev/null + } + start_time=$(date +%s) + echo "Starting file generation..." + + # Generate 1000 files with random sizes between 10KB and 1MB + for file_num in $(seq 1 1000); do + # Generate random size between 10KB and 1MB (in KB) + size_kb=$(( (RANDOM % 990) + 10 )) # Generates 20-1000KB + + # Convert to MB for fio (with two decimal places) + size=$(awk "BEGIN {printf \"%.2f\", $size_kb / 1024}") + + # Show progress every 100 files + if [ $((file_num % 100)) -eq 0 ]; then + echo "Generated $file_num files..." + fi + + generate_file "$file_num" "$size" & + done + + wait + + end_time=$(date +%s) + elapsed_time=$((end_time - start_time)) + echo "File generation complete!" + echo "Total time taken: $elapsed_time seconds" + + # Display summary of generated files (adapted for /data) + total_size=$(du -sh "${OUTPUT_DIR}" | cut -f1) + echo "Total size of generated files: $total_size" + echo "File size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print $5}' | awk '{ sum += $1; n++ } END { print "Average file size: " sum/n/1024/1024 " MB" }' + + # Optional: Show detailed distribution + echo -e "\nDetailed size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print int($5/1024/1024)"MB"}' | sort | uniq -c diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/kustomization.yaml new file mode 100644 index 000000000..28d971b58 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/delete-data/kustomization.yaml @@ -0,0 +1,202 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cleanup-disk.yaml + - configmap.yaml + - ../../base + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=fio-test-parallelstore-sidecar-3k + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-b + - INSTANCE_NAME=parallelstore-us-central1-b + - STORAGE_SIZE=27000Gi + - ACCESS_POINTS=10.93.0.3, 10.93.0.4, 10.93.0.2 + - NETWORK=research-vpc + - MOUNT_LOCALITY=node + - TEST_DIR=test_files + - MOUNT_PATH=/data + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=cleanup].env.[name=OUTPUT_DIR].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=cleanup].volumeMounts.[name=parallelstore-volume].mountPath +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-config.yaml new file mode 100644 index 000000000..db5ef8278 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-config.yaml @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: fio-config-gcs +data: + fio.conf: | + [global] + name=random_files_test + directory=/data + filename_format=random_file.${POD_NAME}.$filenum + ioengine=libaio + direct=1 + group_reporting=1 + fadvise_hint=0 + runtime=300 + time_based=1 + unlink=1 + + [job1] + rw=rw + rwmixread=50 + blocksize=16k + openfiles=1000 + nrfiles=1000 + filesize=10k-30m + thread=1 + + [cleanup] + stonewall + rw=trim diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/fio-config.yaml new file mode 100644 index 000000000..0ebc53a9d --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/fio-config.yaml @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# 5 files (1-5mb) + +apiVersion: v1 +kind: ConfigMap +metadata: + name: fio-config-gcs +data: + fio.conf: | + [global] + bs=4k + direct=1 + ioengine=sync + numjobs=1 + runtime=300 + time_based + group_reporting + invalidate=1 + startdelay=10 + + [randomreads] + name=small_files_random_read_${POD_NAME} + directory=${MOUNT_PATH}/${TEST_DIR} + filename_format=testfile_* + rw=randread + file_service_type=random + openfiles=5 + nrfiles=1000 + randrepeat=0 + filesize=1M-5M + create_on_open=0 + size=0 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/gcs-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/gcs-testing.yaml new file mode 100644 index 000000000..61cdfb444 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/gcs-testing.yaml @@ -0,0 +1,81 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-gcs-30000-1 + labels: + app: fio-test-gcs +spec: + completionMode: Indexed + backoffLimitPerIndex: 10 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-limit: "500m" + gke-gcsfuse/memory-limit: "512Mi" + gke-gcsfuse/ephemeral-storage-limit: "5Gi" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/ephemeral-storage-request: "200Mi" + labels: + app: fio-test-gcs + spec: + priorityClassName: default-priority + nodeSelector: + cloud.google.com/compute-class: spot-capacity + restartPolicy: OnFailure + containers: + - name: fio + image: fio + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: 4Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: /data + - name: TEST_DIR + value: test_files + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONFIG_PATH + value: /etc/fio/fio.conf + volumeMounts: + - name: gcs-fuse-csi-ephemeral + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: gcs-fuse-csi-ephemeral + csi: + driver: gcsfuse.csi.storage.gke.io + volumeAttributes: + bucketName: fsi-research-1-us-central1-gke-data-beao + mountOptions: "implicit-dirs" + gcsfuseLoggingSeverity: warning + - name: fio-config + configMap: + name: fio-config-gcs diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/kustomization.yaml new file mode 100644 index 000000000..e957c0c47 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/fio-test/kustomization.yaml @@ -0,0 +1,101 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- gcs-testing.yaml +- fio-config.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=fio-test-gcs-10k-cache-15k-final-1 + - BUCKETNAME=fsi-research-1-us-central1-gke-data-beao + - COMPLETIONS=10000 + - PARALLELISM=5000 + - TEST_DIR=test_files + - MOUNT_PATH=/data + name: gcs-vars + +replacements: +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.BUCKETNAME + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=gcs-fuse-csi-ephemeral].csi.volumeAttributes.bucketName +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=gcs-fuse-csi-ephemeral].mountPath + +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/gcs-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/gcs-testing.yaml new file mode 100644 index 000000000..fac3c9e36 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/gcs-testing.yaml @@ -0,0 +1,74 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-gcs + labels: + app: fio-test +spec: + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-limit: "200m" + gke-gcsfuse/memory-limit: "256Mi" + gke-gcsfuse/ephemeral-storage-limit: "5Gi" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/ephemeral-storage-request: "200Mi" + labels: + app: fio-test-gcs + spec: + priorityClassName: default-priority + nodeSelector: + cloud.google.com/compute-class: spot-capacity + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "1" + memory: 4Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: "/data" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumeMounts: + - name: gcs-fuse-csi-ephemeral + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: gcs-fuse-csi-ephemeral + csi: + driver: gcsfuse.csi.storage.gke.io + volumeAttributes: + bucketName: fsi-research-5-fio-dual + mountOptions: "implicit-dirs" + gcsfuseLoggingSeverity: warning + - name: fio-config + configMap: + name: fio-config-gcs diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/generate-data/generate-data-gcs.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/generate-data/generate-data-gcs.yaml new file mode 100644 index 000000000..c128ebf43 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/generate-data/generate-data-gcs.yaml @@ -0,0 +1,174 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ConfigMap: file-generator-config +apiVersion: v1 +kind: ConfigMap +metadata: + name: file-generator-config +data: + generate_files.sh: | + #!/bin/bash + + RETRY_COUNTER=0 + MAX_RETRIES=5 + SLEEP_TIME=1 + + # Check if OUTPUT_DIR environment variable is set, otherwise use default + if [ -z "${OUTPUT_DIR}" ]; then + OUTPUT_DIR="/data" # Default to /data inside the container + echo "OUTPUT_DIR not set, using default: ${OUTPUT_DIR}" + fi + + echo "Loading Data in to $OUTPUT_DIR" + echo "Waiting for GCS FUSE mount..." + + OUTPUT_BASE_DIR=$(dirname "${OUTPUT_DIR}") + + while [ $RETRY_COUNTER -lt $MAX_RETRIES ]; do + if mountpoint -q $OUTPUT_BASE_DIR; then + echo "[$(date +%Y-%m-%d_%H:%M:%S)] GCS FUSE mount detected at $OUTPUT_BASE_DIR" + break + fi + echo "[$(date +%Y-%m-%d_%H:%M:%S)] GCS FUSE mount not detected, retrying in $SLEEP_TIME seconds..." + sleep $SLEEP_TIME + COUNTER=$((COUNTER+1)) + SLEEP_TIME=$((SLEEP_TIME * 2)) # Exponential backoff + if [ $RETRY_COUNTER -eq $MAX_RETRIES ]; then + echo "[$(date +%Y-%m-%d_%H:%M:%S)] Failed to detect GCS FUSE mount at $OUTPUT_BASE_DIR after multiple retries" + # Call cleanup function if needed + cleanup # Make sure the 'cleanup' function is defined if you use it. + exit 1 + fi + done + + + # Create or clean directory for test files (using /data) + if [ -d "${OUTPUT_DIR}" ]; then + echo "Cleaning existing directory: ${OUTPUT_DIR}" + rm -rf "${OUTPUT_DIR}"/* + else + echo "Creating directory: ${OUTPUT_DIR}" + fi + mkdir -p "${OUTPUT_DIR}" + + # Function to generate a single file using FIO + generate_file() { + file_num=$1 + size=$2 + fio --name=generate_file_${file_num} \ + --ioengine=sync \ + --rw=write \ + --bs=1m \ + --direct=1 \ + --size=${size}M \ + --filename="${OUTPUT_DIR}/testfile_${file_num}" \ + --thread \ + --group_reporting \ + --minimal + # >/dev/null 2>&1 # Redirect output to /dev/null + } + + echo "Starting file generation..." + + # Generate 1000 files with random sizes between 1-5MB + for file_num in $(seq 1 1000); do + # Generate random size between 1-5 (MB) + size=$(( (RANDOM % 5) + 1 )) + + # Show progress every 100 files + if [ $((file_num % 100)) -eq 0 ]; then + echo "Generated $file_num files..." + fi + + generate_file "$file_num" "$size" & + + # Limit concurrent FIO processes to avoid system overload + # Wait if we have too many background processes + if [ $((file_num % 20)) -eq 0 ]; then + wait + fi + done + + # Wait for any remaining background processes to complete + wait + + echo "File generation complete!" + + # Display summary of generated files (adapted for /data) + total_size=$(du -sh "${OUTPUT_DIR}" | cut -f1) + echo "Total size of generated files: $total_size" + echo "File size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print $5}' | awk '{ sum += $1; n++ } END { print "Average file size: " sum/n/1024/1024 " MB" }' + + # Optional: Show detailed distribution + echo -e "\nDetailed size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print int($5/1024/1024)"MB"}' | sort | uniq -c + +--- +# Job: file-generator-job +apiVersion: batch/v1 +kind: Job +metadata: + name: file-generator-job-gcs +spec: + ttlSecondsAfterFinished: 120 + completions: 1 + parallelism: 1 + template: + metadata: + annotations: + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-limit: "1" + gke-gcsfuse/memory-limit: "1Gi" + gke-gcsfuse/ephemeral-storage-limit: "5Gi" + gke-gcsfuse/cpu-request: "50m" + gke-gcsfuse/memory-request: "64Mi" + gke-gcsfuse/ephemeral-storage-request: "200Mi" + spec: + priorityClassName: higher-priority + nodeSelector: + cloud.google.com/compute-class: spot-capacity + containers: + - name: file-generator + image: us-docker.pkg.dev/fsi-research-1/research-images/fio:latest + command: ["/bin/bash", "/scripts/generate_files.sh"] + env: + - name: OUTPUT_DIR + value: "/data/test_files" + resources: + requests: + memory: "2Gi" + cpu: "1" + limits: + memory: "4Gi" + cpu: "2" + volumeMounts: + - name: script-volume + mountPath: /scripts + - name: gcs-fuse-csi-ephemeral + mountPath: /data + volumes: + - name: script-volume + configMap: + name: file-generator-config + - name: gcs-fuse-csi-ephemeral + csi: + driver: gcsfuse.csi.storage.gke.io + volumeAttributes: + bucketName: fsi-research-1-us-east4-gke-data-a8w9 #Non cache + mountOptions: "implicit-dirs" + gcsfuseLoggingSeverity: warning + restartPolicy: Never + backoffLimit: 4 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/generate-data/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/generate-data/kustomization.yaml new file mode 100644 index 000000000..4afb43d11 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/generate-data/kustomization.yaml @@ -0,0 +1,100 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- gcs-testing.yaml +- fio-config.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=fio-test-gcs-10k-cache-15k-final-1 + - BUCKETNAME=fsi-research-1-us-central1-gke-data-beao + - COMPLETIONS=10000 + - PARALLELISM=5000 + - TEST_DIR=test_files + - MOUNT_PATH=/data + name: gcs-vars + +replacements: +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.BUCKETNAME + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=gcs-fuse-csi-ephemeral].csi.volumeAttributes.bucketName +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=gcs-fuse-csi-ephemeral].mountPath +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/kustomization.yaml new file mode 100644 index 000000000..b391da109 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/gcs/kustomization.yaml @@ -0,0 +1,62 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- gcs-testing.yaml +- fio-config.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/PROJECT_ID/research-images/fio # Replace with your desired registry + newTag: latest + +configMapGenerator: +- literals: + - BUCKETNAME=BUCKETNAME + - COMPLETIONS=1 + - PARALLELISM=1 + name: gcs-vars + +replacements: +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.BUCKETNAME + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=gcs-fuse-csi-ephemeral].csi.volumeAttributes.bucketName + +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: gcs-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/helper.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/helper.yaml new file mode 100644 index 000000000..f87cd3cf9 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/helper.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: helper-pod + namespace: default +spec: + terminationGracePeriodSeconds: 60 + containers: + - image: busybox + name: busybox + command: ["sleep"] + args: ["infinity"] + volumeMounts: + - name: parallelstore-volume + mountPath: /data/parallelstore + # - name: gcs-fuse-csi-ephemeral + # mountPath: /data/gcs + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-us-central1-b + # - name: gcs-fuse-csi-ephemeral + # csi: + # driver: gcsfuse.csi.storage.gke.io + # volumeAttributes: + # bucketName: BUCKETNAME + # mountOptions: "implicit-dirs" + # gcsfuseLoggingSeverity: warning diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/kustomization.yaml new file mode 100644 index 000000000..ee328c5a6 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/kustomization.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - lustre-pvc.yaml + - lustre-pv.yaml diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/lustre-pv.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/lustre-pv.yaml new file mode 100644 index 000000000..441aa9dda --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/lustre-pv.yaml @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolume +metadata: + name: lustre-pv +spec: + storageClassName: "" + capacity: + storage: 16Ti + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + csi: + driver: lustre.csi.storage.gke.io + volumeHandle: PROJECT_ID/LOCATION/INSTANCE_NAME # Modify this to use the project, zone, and managed lustre instance name. + volumeAttributes: + ip: default # The IP address of the existing Lustre instance. + filesystem: default # The filesystem name of the existing Lustre instance. diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/lustre-pvc.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/lustre-pvc.yaml new file mode 100644 index 000000000..5369a10c0 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/base/lustre-pvc.yaml @@ -0,0 +1,26 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: lustre-pvc +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: lustre-pv + resources: + requests: + storage: 16Ti diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/delete-data/cleanup-disk.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/delete-data/cleanup-disk.yaml new file mode 100644 index 000000000..dee48a72e --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/delete-data/cleanup-disk.yaml @@ -0,0 +1,76 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: data-cleanup-job +spec: + completions: 1 + parallelism: 1 + template: + spec: + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: higher-priority + containers: + - name: cleanup + image: busybox + command: ["sh", "-c"] + env: + - name: CLEANUP_DIR + value: test_files + - name: MOUNT_PATH + value: /data + args: + - | + echo "Data cleanup starting..." + if [ -z "${CLEANUP_DIR}" ]; then + echo "CLEANUP_DIR not set, skipping deletion." + else + echo "Deleting contents of: /data/${CLEANUP_DIR}" + # Check if the directory exists before attempting to delete it + if [ -d "/data/${CLEANUP_DIR}" ]; then + rm -rf "/data/${CLEANUP_DIR}" + else + echo "Directory /data/${CLEANUP_DIR} does not exist, skipping deletion." + fi + fi + # Check if the directory exists, and if it does, check if it is empty + if [ -d "/data/${CLEANUP_DIR}" ]; then + file_count=$(find "/data/${CLEANUP_DIR}" -mindepth 1 -maxdepth 1 -type f | wc -l) + if [ "$file_count" -gt 0 ]; then + echo "Error: Files remaining after cleanup. Count: $file_count" + exit 1 + fi + else + echo "Directory /data/${CLEANUP_DIR} does not exist, skipping file count check." + fi + echo "Data cleanup complete." + volumeMounts: + - name: lustre-volume + mountPath: /data + resources: + requests: + memory: "16Gi" + cpu: "4" + limits: + memory: "32Gi" + cpu: "8" + restartPolicy: Never + volumes: + - name: lustre-volume + persistentVolumeClaim: + claimName: lustre-pvc + ttlSecondsAfterFinished: 120 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/delete-data/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/delete-data/kustomization.yaml new file mode 100644 index 000000000..cf1877dbc --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/delete-data/kustomization.yaml @@ -0,0 +1,195 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cleanup-disk.yaml + - configmap.yaml + - ../../base + +images: +- name: cleanup + newName: busybox + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=fio-cleanup + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-a + - INSTANCE_NAME=lustre-us-central1-a + - STORAGE_SIZE=16Ti + - INSTANCE_IP=172.16.1.3 + - FILESYSTEM=testfs + - CLEANUP_DIR=test_files/large_files + - MOUNT_PATH=/data + name: lustre-vars + +replacements: +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.volumeName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.CLEANUP_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=cleanup].env.[name=CLEANUP_DIR].value +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=cleanup].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=cleanup].volumeMounts.[name=lustre-volume].mountPath +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.INSTANCE_IP + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.ip +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.FILESYSTEM + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.filesystem + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=lustre-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/configmap.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/configmap.yaml new file mode 100644 index 000000000..ba513d7c4 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/configmap.yaml @@ -0,0 +1,113 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: file-generator-config +data: + generate_files.sh: | + #!/bin/bash + + MAX_RETRIES=${MAX_RETRIES:-5} + SLEEP_TIME=${SLEEP_TIME:-1} + NUM_FILES=${NUM_FILES:-1000} + MIN_FILE_SIZE_KB=${MIN_FILE_SIZE_KB:-10} + MAX_FILE_SIZE_KB=${MAX_FILE_SIZE_KB:-1000} + OUTPUT_DIR=${OUTPUT_DIR} + + RETRY_COUNTER=0 + + echo "Sleeping for ${SLEEP_TIME} seconds..." + sleep ${SLEEP_TIME} + + # Check if OUTPUT_DIR environment variable is set + if [ -z "${OUTPUT_DIR}" ]; then + echo "ERROR: OUTPUT_DIR not set. Please set it before running the script." + exit 1 + fi + + # Ensure the parent directory exists + if [ ! -d "/data" ]; then + echo "ERROR: /data directory does not exist. Please create it." + exit 1 + fi + + # Create or clean directory for test files + if [ -d "${OUTPUT_DIR}" ]; then + echo "Cleaning existing directory: ${OUTPUT_DIR}" + rm -rf "${OUTPUT_DIR}"/* + fi + + echo "Creating directory: ${OUTPUT_DIR}" + mkdir -p "${OUTPUT_DIR}" + + # Check if directory was created successfully. + if [ ! -d "${OUTPUT_DIR}" ]; then + echo "ERROR: Failed to create directory ${OUTPUT_DIR}" + exit 1 + fi + + # Function to generate a single file using FIO + generate_file() { + file_num=$1 + size=$2 + fio --name=generate_file_${file_num} \ + --ioengine=libaio \ + --rw=write \ + --bs=4k \ + --size=${size} \ + --filename="${OUTPUT_DIR}/testfile_${file_num}" \ + --thread \ + --group_reporting \ + --minimal + } + + start_time=$(date +%s) + echo "Starting file generation..." + + # Generate files with random sizes between MIN_FILE_SIZE_KB and MAX_FILE_SIZE_KB + for file_num in $(seq 1 "${NUM_FILES}"); do + # Generate random size between MIN_FILE_SIZE_KB and MAX_FILE_SIZE_KB (in KB) + size_kb=$(( (RANDOM % (MAX_FILE_SIZE_KB - MIN_FILE_SIZE_KB + 1)) + MIN_FILE_SIZE_KB )) + + # Convert to KB for fio + size="${size_kb}K" + + # Show progress every 100 files + if [ $((file_num % 100)) -eq 0 ]; then + echo "Generated $file_num files..." + fi + + generate_file "$file_num" "$size" # Sequential + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create file testfile_$file_num" + fi + done + + + end_time=$(date +%s) + elapsed_time=$((end_time - start_time)) + echo "File generation complete!" + echo "Total time taken: $elapsed_time seconds" + + # Display summary of generated files + total_size=$(du -sh "${OUTPUT_DIR}" | cut -f1) + echo "Total size of generated files: $total_size" + echo "File size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print $5}' | awk '{ sum += $1; n++ } END { print "Average file size: " sum/n/1024/1024 " MB" }' + + # Optional: Show detailed distribution + echo -e "\nDetailed size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print int($5/1024/1024)"MB"}' | sort | uniq -c diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/generate-data.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/generate-data.yaml new file mode 100644 index 000000000..dbb6add6c --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/generate-data.yaml @@ -0,0 +1,65 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: file-generator-job-lustre +spec: + ttlSecondsAfterFinished: 120 + completions: 1 + parallelism: 1 + template: + spec: + priorityClassName: higher-priority + # nodeSelector: + # cloud.google.com/compute-class: spot-capacity + containers: + - name: file-generator + image: fio + command: ["/bin/bash", "/scripts/generate_files.sh"] + env: + - name: OUTPUT_DIR + value: "/data/test_files" # Default output directory + - name: MAX_RETRIES + value: "5" # Number of retries for mount detection + - name: SLEEP_TIME + value: "1" # Initial sleep time in seconds + - name: NUM_FILES + value: "1000" # Number of files to generate + - name: MIN_FILE_SIZE_KB + value: "1000000" # Minimum file size in KB + - name: MAX_FILE_SIZE_KB + value: "10000000" # Maximum file size in KB + resources: + requests: + memory: "8Gi" + cpu: "4" + limits: + memory: "16Gi" + cpu: "8" + volumeMounts: + - name: script-volume + mountPath: /scripts + - name: lustre-volume + mountPath: /data + volumes: + - name: script-volume + configMap: + name: file-generator-config + - name: lustre-volume + persistentVolumeClaim: + claimName: lustre-pvc + restartPolicy: Never + backoffLimit: 4 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/kustomization.yaml new file mode 100644 index 000000000..23298e349 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/generate-data/kustomization.yaml @@ -0,0 +1,202 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - generate-data.yaml + - configmap.yaml + - ../../base + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=generate-data-lustre-small-files + - COMPLETIONS=1 + - PARALLELISM=1 + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-a + - INSTANCE_NAME=lustre-us-central1-a + - STORAGE_SIZE=16Ti + - INSTANCE_IP=172.16.1.3 + - FILESYSTEM=testfs + # - OUTPUT_DIR=/data/test_files/large_files + - OUTPUT_DIR=/data/test_files/small_files + - MAX_RETRIES=5 + - SLEEP_TIME=10 + - NUM_FILES=1000 + - MIN_FILE_SIZE_KB=50 + - MAX_FILE_SIZE_KB=5000 + name: lustre-vars + +replacements: +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.volumeName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.INSTANCE_IP + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.ip +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.FILESYSTEM + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.filesystem + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=lustre-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.OUTPUT_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=file-generator].env.[name=OUTPUT_DIR].value diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/fio-config.yaml new file mode 100644 index 000000000..01c48e9b2 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/fio-config.yaml @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: lustre-fio-config +data: + fio.conf: | + [global] + directory=${MOUNT_PATH}/${TEST_DIR} + bs=4k + ioengine=libaio + iodepth=1 + time_based + runtime=600 + group_reporting + invalidate=1 + size=100G + + # Large files + [parallel_reads] + name=library_parallel_reads_${POD_NAME} + filename_format=testfile_* + rw=randread + numjobs=1 + nrfiles=125 + openfiles=100 + filesize=50m-250m + file_service_type=random + randrepeat=0 + norandommap=1 + startdelay=10 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/kustomization.yaml new file mode 100644 index 000000000..3a2b1126d --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/kustomization.yaml @@ -0,0 +1,215 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base/ +- fio-config.yaml +- lustre-testing.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=lustre-large-files-1 + - COMPLETIONS=3000 + - PARALLELISM=3000 + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-a + - INSTANCE_NAME=lustre-us-central1-a + - STORAGE_SIZE=16Ti + - INSTANCE_IP=172.16.1.3 + - FILESYSTEM=testfs + - TEST_DIR=test_files/large_files + - MOUNT_PATH=/data + name: lustre-vars + +replacements: +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.volumeName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=lustre-volume].mountPath +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.INSTANCE_IP + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.ip +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.FILESYSTEM + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.filesystem + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=lustre-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/lustre-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/lustre-testing.yaml new file mode 100644 index 000000000..84198d86a --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/large-files-test/lustre-testing.yaml @@ -0,0 +1,69 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-lustre + labels: + app: fio-test-lustre +spec: + completionMode: Indexed + backoffLimitPerIndex: 10 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + labels: + app: fio-test-lustre + spec: + terminationGracePeriodSeconds: 60 + # nodeSelector: + # cloud.google.com/compute-class: spot-capacity + priorityClassName: default-priority + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "2" + memory: 8Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: /data + - name: TEST_DIR + value: test_files + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONFIG_PATH + value: /etc/fio/fio.conf + volumeMounts: + - name: lustre-volume + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: lustre-volume + persistentVolumeClaim: + claimName: lustre-pvc + - name: fio-config + configMap: + name: lustre-fio-config diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/fio-config.yaml new file mode 100644 index 000000000..cf7d0e43c --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/fio-config.yaml @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: parallelstore-fio-config +data: + fio.conf: | + [global] + directory=${MOUNT_PATH}/${TEST_DIR} + bs=4k + direct=1 + ioengine=libaio + iodepth=32 + time_based + runtime=300 + group_reporting + invalidate=1 + size=100G + + # Small Reads + [parallel_reads] + name=library_parallel_reads_${POD_NAME} + filename_format=testfile_* + rw=randread + numjobs=32 + nrfiles=1000 + openfiles=100 + filesize=50k-1M + file_service_type=random + randrepeat=0 + norandommap=1 + startdelay=10 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/kustomization.yaml new file mode 100644 index 000000000..35e8221a9 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/kustomization.yaml @@ -0,0 +1,215 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base/ +- fio-config.yaml +- lustre-testing.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=lustre-run-1-small + - COMPLETIONS=1 + - PARALLELISM=1 + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-a + - INSTANCE_NAME=lustre-us-central1-a + - STORAGE_SIZE=16Ti + - INSTANCE_IP=172.16.1.3 + - FILESYSTEM=testfs + - TEST_DIR=test_files/small_files + - MOUNT_PATH=/data + name: lustre-vars + +replacements: +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.volumeName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=lustre-volume].mountPath +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.INSTANCE_IP + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.ip +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.FILESYSTEM + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.filesystem + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=lustre-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: lustre-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/lustre-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/lustre-testing.yaml new file mode 100644 index 000000000..84198d86a --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/lustre/overlays/small-files-test/lustre-testing.yaml @@ -0,0 +1,69 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-lustre + labels: + app: fio-test-lustre +spec: + completionMode: Indexed + backoffLimitPerIndex: 10 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + labels: + app: fio-test-lustre + spec: + terminationGracePeriodSeconds: 60 + # nodeSelector: + # cloud.google.com/compute-class: spot-capacity + priorityClassName: default-priority + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "2" + memory: 8Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: /data + - name: TEST_DIR + value: test_files + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONFIG_PATH + value: /etc/fio/fio.conf + volumeMounts: + - name: lustre-volume + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: lustre-volume + persistentVolumeClaim: + claimName: lustre-pvc + - name: fio-config + configMap: + name: lustre-fio-config diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/kustomization.yaml new file mode 100644 index 000000000..4f5db9887 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/kustomization.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - parallelstore-pvc.yaml + - parallelstore-pv.yaml diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/parallelstore-pv.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/parallelstore-pv.yaml new file mode 100644 index 000000000..86dcc820c --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/parallelstore-pv.yaml @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolume +metadata: + name: parallelstore-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 1Gi + csi: + driver: parallelstore.csi.storage.gke.io + volumeAttributes: + accessPoints: default + mountLocality: default + network: default + volumeHandle: "PROJECT_ID/LOCATION/INSTANCE_NAME/default-pool/default-container" + persistentVolumeReclaimPolicy: Retain + storageClassName: parallelstore-sc + volumeMode: Filesystem diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/parallelstore-pvc.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/parallelstore-pvc.yaml new file mode 100644 index 000000000..add446845 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/base/parallelstore-pvc.yaml @@ -0,0 +1,25 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: parallelstore-pvc +spec: + accessModes: + - ReadWriteMany + storageClassName: parallelstore-sc + resources: + requests: + storage: 21000Gi diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/fio-config.yaml new file mode 100644 index 000000000..35d380a8f --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/fio-config.yaml @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: parallelstore-fio-config +data: + fio.conf: | + [global] + name=random_files_test + directory=/data + filename_format=random_file.${POD_NAME}.$filenum + ioengine=libaio + direct=1 + group_reporting=1 + fadvise_hint=0 + runtime=300 + time_based=1 + unlink=1 + + [job1] + rw=rw + rwmixread=50 + blocksize=16k + openfiles=1000 + nrfiles=1000 + filesize=10k-30m + thread=1 + + [cleanup] + stonewall + rw=trim diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/kustomization.yaml new file mode 100644 index 000000000..d59d0bdf7 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/kustomization.yaml @@ -0,0 +1,208 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- parallelstore-pv.yaml +- parallelstore-pvc.yaml +- parallelstore-testing.yaml +- fio-config.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/PROJECT_ID/research-images/fio # Replace with your desired registry + newTag: latest + + + +configMapGenerator: +- literals: + - STORAGECLASS=STORAGECLASS + - PROJECT_ID=PROJECT_ID + - LOCATION=LOCATION + - INSTANCE_NAME=INSTANCE_NAME + - STORAGE_SIZE=STORAGE_SIZE + - ACCESS_POINTS=ACCESS_POINTS + - NETWORK=research-vpc + - MOUNT_LOCALITY=node + - COMPLETIONS=1000 + - PARALLELISM=1000 + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 + + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGECLASS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGECLASS + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 0 + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 0 + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/cleanup-disk.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/cleanup-disk.yaml new file mode 100644 index 000000000..844189922 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/cleanup-disk.yaml @@ -0,0 +1,74 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: data-cleanup-job +spec: + completions: 1 + parallelism: 1 + template: + spec: + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: higher-priority + containers: + - name: cleanup + image: busybox + command: ["sh", "-c"] + env: + - name: OUTPUT_DIR + value: test_files + args: + - | + echo "Data cleanup starting..." + if [ -z "${OUTPUT_DIR}" ]; then + echo "OUTPUT_DIR not set, skipping deletion." + else + echo "Deleting contents of: /data/${OUTPUT_DIR}" + # Check if the directory exists before attempting to delete it + if [ -d "/data/${OUTPUT_DIR}" ]; then + rm -rf "/data/${OUTPUT_DIR}" + else + echo "Directory /data/${OUTPUT_DIR} does not exist, skipping deletion." + fi + fi + # Check if the directory exists, and if it does, check if it is empty + if [ -d "/data/${OUTPUT_DIR}" ]; then + file_count=$(find "/data/${OUTPUT_DIR}" -mindepth 1 -maxdepth 1 -type f | wc -l) + if [ "$file_count" -gt 0 ]; then + echo "Error: Files remaining after cleanup. Count: $file_count" + exit 1 + fi + else + echo "Directory /data/${OUTPUT_DIR} does not exist, skipping file count check." + fi + echo "Data cleanup complete." + volumeMounts: + - name: parallelstore-volume + mountPath: /data + resources: + requests: + memory: "16Gi" + cpu: "4" + limits: + memory: "32Gi" + cpu: "8" + restartPolicy: Never + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc + ttlSecondsAfterFinished: 120 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/configmap.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/configmap.yaml new file mode 100644 index 000000000..4061968eb --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/configmap.yaml @@ -0,0 +1,116 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: file-generator-config +data: + generate_files.sh: | + #!/bin/bash + + RETRY_COUNTER=0 + MAX_RETRIES=5 + SLEEP_TIME=1 + + # Check if OUTPUT_DIR environment variable is set, otherwise use default + if [ -z "${OUTPUT_DIR}" ]; then + OUTPUT_DIR="/data" # Default to /data inside the container + echo "OUTPUT_DIR not set, using default: ${OUTPUT_DIR}" + fi + + echo "Loading Data in to $OUTPUT_DIR" + echo "Waiting for Parallelstore mount..." + + OUTPUT_BASE_DIR=$(dirname "${OUTPUT_DIR}") + + while [ $RETRY_COUNTER -lt $MAX_RETRIES ]; do + if mountpoint -q $OUTPUT_BASE_DIR; then + echo "[$(date +%Y-%m-%d_%H:%M:%S)] Parallelstore mount detected at $OUTPUT_BASE_DIR" + break + fi + echo "[$(date +%Y-%m-%d_%H:%M:%S)] Parallelstore mount not detected, retrying in $SLEEP_TIME seconds..." + sleep $SLEEP_TIME + COUNTER=$((COUNTER+1)) + SLEEP_TIME=$((SLEEP_TIME * 2)) # Exponential backoff + if [ $RETRY_COUNTER -eq $MAX_RETRIES ]; then + echo "[$(date +%Y-%m-%d_%H:%M:%S)] Failed to detect Parallelstore mount at $OUTPUT_BASE_DIR after multiple retries" + # Call cleanup function if needed + cleanup # Make sure the 'cleanup' function is defined if you use it. + exit 1 + fi + done + + + # Create or clean directory for test files (using /data) + if [ -d "${OUTPUT_DIR}" ]; then + echo "Cleaning existing directory: ${OUTPUT_DIR}" + rm -rf "${OUTPUT_DIR}"/* + else + echo "Creating directory: ${OUTPUT_DIR}" + fi + mkdir -p "${OUTPUT_DIR}" + + # Function to generate a single file using FIO + generate_file() { + file_num=$1 + size=$2 + size_arg="${size_kb}K" + fio --name=generate_file_${file_num} \ + --ioengine=libaio \ + --rw=write \ + --bs=4k \ + --direct=1 \ + --size=${size_arg} \ + --filename="${OUTPUT_DIR}/testfile_${file_num}" \ + --thread \ + --group_reporting \ + --minimal + # >/dev/null 2>&1 # Redirect output to /dev/null + } + start_time=$(date +%s) + echo "Starting file generation..." + + # Generate 1000 files with random sizes between 10KB and 1MB + for file_num in $(seq 1 1000); do + # Generate random size between 10KB and 1MB (in KB) + size_kb=$(( (RANDOM % 990) + 10 )) # Generates 20-1000KB + + # Convert to MB for fio (with two decimal places) + size=$(awk "BEGIN {printf \"%.2f\", $size_kb / 1024}") + + # Show progress every 100 files + if [ $((file_num % 100)) -eq 0 ]; then + echo "Generated $file_num files..." + fi + + generate_file "$file_num" "$size" & + done + + wait + + end_time=$(date +%s) + elapsed_time=$((end_time - start_time)) + echo "File generation complete!" + echo "Total time taken: $elapsed_time seconds" + + # Display summary of generated files (adapted for /data) + total_size=$(du -sh "${OUTPUT_DIR}" | cut -f1) + echo "Total size of generated files: $total_size" + echo "File size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print $5}' | awk '{ sum += $1; n++ } END { print "Average file size: " sum/n/1024/1024 " MB" }' + + # Optional: Show detailed distribution + echo -e "\nDetailed size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print int($5/1024/1024)"MB"}' | sort | uniq -c diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/kustomization.yaml new file mode 100644 index 000000000..5c033083c --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/delete-data/kustomization.yaml @@ -0,0 +1,202 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cleanup-disk.yaml + - configmap.yaml + - ../../base + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=fio-cleanup + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-b + - INSTANCE_NAME=parallelstore-us-central1-b + - STORAGE_SIZE=21000Gi + - ACCESS_POINTS=10.93.0.3, 10.93.0.4, 10.93.0.2 + - NETWORK=research-vpc + - MOUNT_LOCALITY=node + - TEST_DIR=test_files + - MOUNT_PATH=/data + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=cleanup].env.[name=OUTPUT_DIR].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=cleanup].volumeMounts.[name=parallelstore-volume].mountPath +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/fio-config.yaml new file mode 100644 index 000000000..233f550fa --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/fio-config.yaml @@ -0,0 +1,67 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: parallelstore-fio-config +data: + fio.conf: | + [global] + directory=${MOUNT_PATH}/${TEST_DIR} + bs=4k + direct=1 + ioengine=libaio + iodepth=32 + time_based + runtime=300 + group_reporting + invalidate=1 + size=100G + # Small Reads + [parallel_reads] + name=library_parallel_reads_${POD_NAME} + filename_format=testfile_* + rw=randread + numjobs=32 + nrfiles=1000 + openfiles=100 + filesize=50k-1M + file_service_type=random + randrepeat=0 + norandommap=1 + startdelay=10 + # Large files + # [parallel_reads] + # name=library_parallel_reads_${POD_NAME} + # filename_format=testfile_* + # rw=randread + # numjobs=32 + # nrfiles=125 + # openfiles=100 + # filesize=50m-250m + # file_service_type=random + # randrepeat=0 + # norandommap=1 + # startdelay=10 + + # [sequential_reads] + # stonewall + # name=library_sequential_reads_${POD_NAME} + # filename_format=testfile_* + # rw=read + # numjobs=16 + # openfiles=100 + # nrfiles=1000 + # filesize=10k-1M diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/kustomization.yaml new file mode 100644 index 000000000..ced32e646 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/kustomization.yaml @@ -0,0 +1,234 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base/ +- fio-config.yaml +- parallelstore-testing.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=fio-test-parallelstore-smallfiles-5k-a + - COMPLETIONS=50000 + - PARALLELISM=30000 + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-b + - INSTANCE_NAME=parallelstore-us-central1-b + - STORAGE_SIZE=27000Gi + - ACCESS_POINTS=10.93.0.3, 10.93.0.4, 10.93.0.2 + - NETWORK=research-vpc + - MOUNT_LOCALITY=node + - TEST_DIR=test_files/small_files + - MOUNT_PATH=/data + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=parallelstore-volume].mountPath +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/parallelstore-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/parallelstore-testing.yaml new file mode 100644 index 000000000..8a248beeb --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/fio-test/parallelstore-testing.yaml @@ -0,0 +1,77 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-parallelstore + labels: + app: fio-test-parallelstore +spec: + completionMode: Indexed + backoffLimitPerIndex: 10 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + # annotations: + # gke-parallelstore/volumes: "true" + # gke-parallelstore/cpu-request: 500m + # gke-parallelstore/memory-request: 1Gi + # gke-parallelstore/ephemeral-storage-request: 500Mi + # gke-parallelstore/cpu-limit: 1000m + # gke-parallelstore/memory-limit: 2Gi + # gke-parallelstore/ephemeral-storage-limit: 1Gi + labels: + app: fio-test-parallelstore + spec: + terminationGracePeriodSeconds: 60 + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: default-priority + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "2" + memory: 8Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: /data + - name: TEST_DIR + value: test_files + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONFIG_PATH + value: /etc/fio/fio.conf + volumeMounts: + - name: parallelstore-volume + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc + - name: fio-config + configMap: + name: parallelstore-fio-config diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/configmap.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/configmap.yaml new file mode 100644 index 000000000..ba513d7c4 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/configmap.yaml @@ -0,0 +1,113 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: file-generator-config +data: + generate_files.sh: | + #!/bin/bash + + MAX_RETRIES=${MAX_RETRIES:-5} + SLEEP_TIME=${SLEEP_TIME:-1} + NUM_FILES=${NUM_FILES:-1000} + MIN_FILE_SIZE_KB=${MIN_FILE_SIZE_KB:-10} + MAX_FILE_SIZE_KB=${MAX_FILE_SIZE_KB:-1000} + OUTPUT_DIR=${OUTPUT_DIR} + + RETRY_COUNTER=0 + + echo "Sleeping for ${SLEEP_TIME} seconds..." + sleep ${SLEEP_TIME} + + # Check if OUTPUT_DIR environment variable is set + if [ -z "${OUTPUT_DIR}" ]; then + echo "ERROR: OUTPUT_DIR not set. Please set it before running the script." + exit 1 + fi + + # Ensure the parent directory exists + if [ ! -d "/data" ]; then + echo "ERROR: /data directory does not exist. Please create it." + exit 1 + fi + + # Create or clean directory for test files + if [ -d "${OUTPUT_DIR}" ]; then + echo "Cleaning existing directory: ${OUTPUT_DIR}" + rm -rf "${OUTPUT_DIR}"/* + fi + + echo "Creating directory: ${OUTPUT_DIR}" + mkdir -p "${OUTPUT_DIR}" + + # Check if directory was created successfully. + if [ ! -d "${OUTPUT_DIR}" ]; then + echo "ERROR: Failed to create directory ${OUTPUT_DIR}" + exit 1 + fi + + # Function to generate a single file using FIO + generate_file() { + file_num=$1 + size=$2 + fio --name=generate_file_${file_num} \ + --ioengine=libaio \ + --rw=write \ + --bs=4k \ + --size=${size} \ + --filename="${OUTPUT_DIR}/testfile_${file_num}" \ + --thread \ + --group_reporting \ + --minimal + } + + start_time=$(date +%s) + echo "Starting file generation..." + + # Generate files with random sizes between MIN_FILE_SIZE_KB and MAX_FILE_SIZE_KB + for file_num in $(seq 1 "${NUM_FILES}"); do + # Generate random size between MIN_FILE_SIZE_KB and MAX_FILE_SIZE_KB (in KB) + size_kb=$(( (RANDOM % (MAX_FILE_SIZE_KB - MIN_FILE_SIZE_KB + 1)) + MIN_FILE_SIZE_KB )) + + # Convert to KB for fio + size="${size_kb}K" + + # Show progress every 100 files + if [ $((file_num % 100)) -eq 0 ]; then + echo "Generated $file_num files..." + fi + + generate_file "$file_num" "$size" # Sequential + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create file testfile_$file_num" + fi + done + + + end_time=$(date +%s) + elapsed_time=$((end_time - start_time)) + echo "File generation complete!" + echo "Total time taken: $elapsed_time seconds" + + # Display summary of generated files + total_size=$(du -sh "${OUTPUT_DIR}" | cut -f1) + echo "Total size of generated files: $total_size" + echo "File size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print $5}' | awk '{ sum += $1; n++ } END { print "Average file size: " sum/n/1024/1024 " MB" }' + + # Optional: Show detailed distribution + echo -e "\nDetailed size distribution:" + ls -l "${OUTPUT_DIR}" | awk '{print int($5/1024/1024)"MB"}' | sort | uniq -c diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/generate-data.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/generate-data.yaml new file mode 100644 index 000000000..1ebf9f1a9 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/generate-data.yaml @@ -0,0 +1,65 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: file-generator-job-parallelstore +spec: + ttlSecondsAfterFinished: 120 + completions: 1 + parallelism: 1 + template: + spec: + priorityClassName: higher-priority + nodeSelector: + cloud.google.com/compute-class: spot-capacity + containers: + - name: file-generator + image: fio + command: ["/bin/bash", "/scripts/generate_files.sh"] + env: + - name: OUTPUT_DIR + value: "/data/test_files" # Default output directory + - name: MAX_RETRIES + value: "5" # Number of retries for mount detection + - name: SLEEP_TIME + value: "1" # Initial sleep time in seconds + - name: NUM_FILES + value: "1000" # Number of files to generate + - name: MIN_FILE_SIZE_KB + value: "1000000" # Minimum file size in KB + - name: MAX_FILE_SIZE_KB + value: "10000000" # Maximum file size in KB + resources: + requests: + memory: "8Gi" + cpu: "4" + limits: + memory: "16Gi" + cpu: "8" + volumeMounts: + - name: script-volume + mountPath: /scripts + - name: parallelstore-volume + mountPath: /data + volumes: + - name: script-volume + configMap: + name: file-generator-config + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc + restartPolicy: Never + backoffLimit: 4 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/kustomization.yaml new file mode 100644 index 000000000..e6776a3f1 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/generate-data/kustomization.yaml @@ -0,0 +1,242 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - generate-data.yaml + - configmap.yaml + - ../../base + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=generate-data-parallelstore-large-files + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-b + - INSTANCE_NAME=parallelstore-us-central1-b + - STORAGE_SIZE=27000Gi + - ACCESS_POINTS=10.93.0.3, 10.93.0.4, 10.93.0.2 + - NETWORK=research-vpc + - MOUNT_LOCALITY=node + - OUTPUT_DIR=/data/test_files/large_files + - MAX_RETRIES=5 + - SLEEP_TIME=10 + - NUM_FILES=1000 + - MIN_FILE_SIZE_KB=50000 + - MAX_FILE_SIZE_KB=250000 + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MAX_RETRIES + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=file-generator].env.[name=MAX_RETRIES].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.SLEEP_TIME + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=file-generator].env.[name=SLEEP_TIME].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NUM_FILES + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=file-generator].env.[name=NUM_FILES].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MIN_FILE_SIZE_KB + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=file-generator].env.[name=MIN_FILE_SIZE_KB].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MAX_FILE_SIZE_KB + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=file-generator].env.[name=MAX_FILE_SIZE_KB].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.OUTPUT_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=file-generator].env.[name=OUTPUT_DIR].value diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/fio-config.yaml new file mode 100644 index 000000000..19d398e1e --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/fio-config.yaml @@ -0,0 +1,44 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: parallelstore-fio-config +data: + fio.conf: | + [global] + directory=${MOUNT_PATH}/${TEST_DIR} + bs=4k + ioengine=libaio + iodepth=1 + time_based + runtime=600 + group_reporting + invalidate=1 + size=100G + + # Large files + [parallel_reads] + name=library_parallel_reads_${POD_NAME} + filename_format=testfile_* + rw=randread + numjobs=1 + nrfiles=125 + openfiles=100 + filesize=50m-250m + file_service_type=random + randrepeat=0 + norandommap=1 + startdelay=10 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/kustomization.yaml new file mode 100644 index 000000000..0b2fa210e --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/kustomization.yaml @@ -0,0 +1,234 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base/ +- fio-config.yaml +- parallelstore-testing.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=500-large-files + - COMPLETIONS=50000 + - PARALLELISM=30000 + - PROJECT_ID=fsi-research-1 + - LOCATION=us-east4-b + - INSTANCE_NAME=parallelstore-us-east4-b + - STORAGE_SIZE=12000Gi + - ACCESS_POINTS=172.16.0.132, 172.16.0.131, 172.16.0.130 + - NETWORK=research-vpc + - MOUNT_LOCALITY=node + - TEST_DIR=test_files/large_files + - MOUNT_PATH=/data + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=parallelstore-volume].mountPath +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/parallelstore-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/parallelstore-testing.yaml new file mode 100644 index 000000000..5dca2f52b --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-node/parallelstore-testing.yaml @@ -0,0 +1,69 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-parallelstore + labels: + app: fio-test-parallelstore +spec: + completionMode: Indexed + backoffLimitPerIndex: 10 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + labels: + app: fio-test-parallelstore + spec: + terminationGracePeriodSeconds: 60 + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: default-priority + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "2" + memory: 8Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: /data + - name: TEST_DIR + value: test_files + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONFIG_PATH + value: /etc/fio/fio.conf + volumeMounts: + - name: parallelstore-volume + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc + - name: fio-config + configMap: + name: parallelstore-fio-config diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/fio-config.yaml new file mode 100644 index 000000000..34be28fa3 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/fio-config.yaml @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: parallelstore-fio-config +data: + fio.conf: | + [global] + directory=${MOUNT_PATH}/${TEST_DIR} + bs=4k + direct=1 + ioengine=libaio + iodepth=32 + time_based + runtime=300 + group_reporting + invalidate=1 + size=100G + + Large files + [parallel_reads] + name=library_parallel_reads_${POD_NAME} + filename_format=testfile_* + rw=randread + numjobs=32 + nrfiles=125 + openfiles=100 + filesize=50m-250m + file_service_type=random + randrepeat=0 + norandommap=1 + startdelay=10 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/kustomization.yaml new file mode 100644 index 000000000..9541617bb --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/kustomization.yaml @@ -0,0 +1,234 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base/ +- fio-config.yaml +- parallelstore-testing.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=fio-test-parallelstore-largefiles-3k-sidecar + - COMPLETIONS=5000 + - PARALLELISM=3000 + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-b + - INSTANCE_NAME=parallelstore-us-central1-b + - STORAGE_SIZE=27000Gi + - ACCESS_POINTS=10.93.0.3, 10.93.0.4, 10.93.0.2 + - NETWORK=research-vpc + - MOUNT_LOCALITY=pod + - TEST_DIR=test_files/large_files + - MOUNT_PATH=/data + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=parallelstore-volume].mountPath +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/parallelstore-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/parallelstore-testing.yaml new file mode 100644 index 000000000..4f92f4793 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/parallelstore-testing.yaml @@ -0,0 +1,77 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-parallelstore + labels: + app: fio-test-parallelstore +spec: + completionMode: Indexed + backoffLimitPerIndex: 10 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + annotations: + gke-parallelstore/volumes: "true" + gke-parallelstore/cpu-request: 500m + gke-parallelstore/memory-request: 1Gi + gke-parallelstore/ephemeral-storage-request: 500Mi + gke-parallelstore/cpu-limit: 1000m + gke-parallelstore/memory-limit: 2Gi + gke-parallelstore/ephemeral-storage-limit: 1Gi + labels: + app: fio-test-parallelstore + spec: + terminationGracePeriodSeconds: 60 + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: default-priority + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "2" + memory: 8Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: /data + - name: TEST_DIR + value: test_files + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONFIG_PATH + value: /etc/fio/fio.conf + volumeMounts: + - name: parallelstore-volume + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc + - name: fio-config + configMap: + name: parallelstore-fio-config diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/fio-config.yaml new file mode 100644 index 000000000..cf7d0e43c --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/fio-config.yaml @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: parallelstore-fio-config +data: + fio.conf: | + [global] + directory=${MOUNT_PATH}/${TEST_DIR} + bs=4k + direct=1 + ioengine=libaio + iodepth=32 + time_based + runtime=300 + group_reporting + invalidate=1 + size=100G + + # Small Reads + [parallel_reads] + name=library_parallel_reads_${POD_NAME} + filename_format=testfile_* + rw=randread + numjobs=32 + nrfiles=1000 + openfiles=100 + filesize=50k-1M + file_service_type=random + randrepeat=0 + norandommap=1 + startdelay=10 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/kustomization.yaml new file mode 100644 index 000000000..7a584ad5a --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/kustomization.yaml @@ -0,0 +1,235 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base/ +- fio-config.yaml +- parallelstore-testing.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + # - JOB_NAME=fio-test-parallelstore-smallfiles-3k-node + - JOB_NAME=test-node + - COMPLETIONS=5 + - PARALLELISM=3 + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-b + - INSTANCE_NAME=parallelstore-us-central1-b + - STORAGE_SIZE=27000Gi + - ACCESS_POINTS=10.93.0.3, 10.93.0.4, 10.93.0.2 + - NETWORK=research-vpc + - MOUNT_LOCALITY=node + - TEST_DIR=test_files/small_files + - MOUNT_PATH=/data + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=parallelstore-volume].mountPath +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/parallelstore-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/parallelstore-testing.yaml new file mode 100644 index 000000000..5dca2f52b --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-node/parallelstore-testing.yaml @@ -0,0 +1,69 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-parallelstore + labels: + app: fio-test-parallelstore +spec: + completionMode: Indexed + backoffLimitPerIndex: 10 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + labels: + app: fio-test-parallelstore + spec: + terminationGracePeriodSeconds: 60 + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: default-priority + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "2" + memory: 8Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: /data + - name: TEST_DIR + value: test_files + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONFIG_PATH + value: /etc/fio/fio.conf + volumeMounts: + - name: parallelstore-volume + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc + - name: fio-config + configMap: + name: parallelstore-fio-config diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/fio-config.yaml new file mode 100644 index 000000000..2ed64c791 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/fio-config.yaml @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: parallelstore-fio-config +data: + fio.conf: | + [global] + directory=${MOUNT_PATH}/${TEST_DIR} + bs=4k + # direct=1 + ioengine=libaio + iodepth=1 + time_based + runtime=300 + group_reporting + invalidate=1 + size=100G + + # Small Reads + [parallel_reads] + name=library_parallel_reads_${POD_NAME} + filename_format=testfile_* + rw=randread + numjobs=1 + nrfiles=1000 + openfiles=100 + filesize=50k-1M + file_service_type=random + randrepeat=0 + norandommap=1 + startdelay=10 diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/kustomization.yaml new file mode 100644 index 000000000..f9036d459 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/kustomization.yaml @@ -0,0 +1,234 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base/ +- fio-config.yaml +- parallelstore-testing.yaml + +images: +- name: fio + newName: us-docker.pkg.dev/fsi-research-1/research-images/fio + newTag: latest + +configMapGenerator: +- literals: + - JOB_NAME=3k-sidecar + - COMPLETIONS=5000 + - PARALLELISM=3000 + - PROJECT_ID=fsi-research-1 + - LOCATION=us-central1-b + - INSTANCE_NAME=parallelstore-us-central1-b + - STORAGE_SIZE=27000Gi + - ACCESS_POINTS=10.93.0.3, 10.93.0.4, 10.93.0.2 + - NETWORK=research-vpc + - MOUNT_LOCALITY=pod + - TEST_DIR=test_files/small_files + - MOUNT_PATH=/data + name: parallelstore-vars + +replacements: +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.JOB_NAME + targets: + - select: + kind: Job + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.STORAGE_SIZE + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.capacity.storage + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.resources.requests.storage +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.TEST_DIR + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=TEST_DIR].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].env.[name=MOUNT_PATH].value +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_PATH + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.containers.[name=fio].volumeMounts.[name=parallelstore-volume].mountPath +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.ACCESS_POINTS + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.accessPoints + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.NETWORK + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.network + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.MOUNT_LOCALITY + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeAttributes.mountLocality + +# Volume handle specific replacement +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PROJECT_ID + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 0 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 1 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.INSTANCE_NAME + targets: + - select: + kind: PersistentVolume + fieldPaths: + - spec.csi.volumeHandle + options: + delimiter: '/' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolume + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - metadata.name + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: PersistentVolumeClaim + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + - select: + kind: PersistentVolume + fieldPaths: + - spec.storageClassName + options: + delimiter: '-' + index: 2 + +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.LOCATION + targets: + - select: + kind: Job + fieldPaths: + - spec.template.spec.volumes.[name=parallelstore-volume].persistentVolumeClaim.claimName + options: + delimiter: '-' + index: 2 +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.COMPLETIONS + targets: + - select: + kind: Job + fieldPaths: + - spec.completions +- source: + kind: ConfigMap + name: parallelstore-vars + fieldPath: data.PARALLELISM + targets: + - select: + kind: Job + fieldPaths: + - spec.parallelism diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/parallelstore-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/parallelstore-testing.yaml new file mode 100644 index 000000000..4f92f4793 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/small-files-test-sidecar/parallelstore-testing.yaml @@ -0,0 +1,77 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-parallelstore + labels: + app: fio-test-parallelstore +spec: + completionMode: Indexed + backoffLimitPerIndex: 10 + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + annotations: + gke-parallelstore/volumes: "true" + gke-parallelstore/cpu-request: 500m + gke-parallelstore/memory-request: 1Gi + gke-parallelstore/ephemeral-storage-request: 500Mi + gke-parallelstore/cpu-limit: 1000m + gke-parallelstore/memory-limit: 2Gi + gke-parallelstore/ephemeral-storage-limit: 1Gi + labels: + app: fio-test-parallelstore + spec: + terminationGracePeriodSeconds: 60 + nodeSelector: + cloud.google.com/compute-class: spot-capacity + priorityClassName: default-priority + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "2" + memory: 8Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: /data + - name: TEST_DIR + value: test_files + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONFIG_PATH + value: /etc/fio/fio.conf + volumeMounts: + - name: parallelstore-volume + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: parallelstore-pvc + - name: fio-config + configMap: + name: parallelstore-fio-config diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-pv.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-pv.yaml new file mode 100644 index 000000000..1cdd279f8 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-pv.yaml @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 1Gi + csi: + driver: parallelstore.csi.storage.gke.io + volumeAttributes: + accessPoints: default + mountLocality: default + network: default + volumeHandle: "PROJECT_ID/REGION-ZONE/INSTANCE_NAME/default-pool/default-container" + persistentVolumeReclaimPolicy: Retain + storageClassName: parallelstore-sc + volumeMode: Filesystem diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-pvc.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-pvc.yaml new file mode 100644 index 000000000..907ebb74b --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-pvc.yaml @@ -0,0 +1,25 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: parallelstore-sc diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-testing.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-testing.yaml new file mode 100644 index 000000000..9256df525 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/parallelstore-testing.yaml @@ -0,0 +1,60 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: fio-test-parallelstore + labels: + app: fio-test +spec: + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 120 + template: + metadata: + labels: + app: fio-test-parallelstore + spec: + priorityClassName: default-priority + restartPolicy: Never + containers: + - name: fio + image: fio + resources: + limits: + cpu: "1" + memory: 8Gi + requests: + cpu: "1" + memory: 4Gi + env: + - name: MOUNT_PATH + value: "/data" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumeMounts: + - name: parallelstore-volume + mountPath: /data + - name: fio-config + mountPath: /etc/fio + volumes: + - name: parallelstore-volume + persistentVolumeClaim: + claimName: pvc + - name: fio-config + configMap: + name: parallelstore-fio-config diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/README.md b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/README.md new file mode 100644 index 000000000..c4f3e4784 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/README.md @@ -0,0 +1,111 @@ +# GCSFUSE Node-Level Mount DaemonSet +> **WARNING:** This is not an officially supported Google service. The use of this solution is on an "as-is" basis and is not a service offered under the Google Cloud Terms of Service. + +> **WARNING:** Google does not endorse the PoC approach described in this document for production use. This PoC is intended solely for internal testing to validate the shared mount architecture for GCS. It should only be used for experimental purposes. Please also note that this PoC does not support Node Autoscaling, non-disruptive node upgrades, or DaemonSet updates. Any attempt at node upgrades or DaemonSet updates will require complete downtime for workloads and will result in workload failures if these actions are attempted. + +This repository contains a Kubernetes Daemonset that deploys GCSFUSE on each node of your cluster, providing a shared mount point for accessing Google Cloud Storage (GCS). + +## Overview + +This Daemonset utilizes GCSFUSE to mount a GCS bucket to a directory on each node. The configuration emphasizes performance and caching for high-concurrency, small-file workloads. Key features include: + +* **Node-level mount:** Provides a consistent mount point accessible to all pods on the node. +* **Optimized for small files:** Caching settings are tuned for frequent access to small files, minimizing latency. +* **High concurrency:** Connection limits are increased to handle many concurrent requests. +* **tmpfs for cache and temp:** Utilizes tmpfs for the GCSFUSE cache and temporary directories, leveraging node memory for improved performance. +* **`allow_other`:** Enables access to the mount point from pods running with different user IDs. + +## Configuration + +The `gcsfuse-config` ConfigMap defines the GCSFUSE settings: + +* **`file-cache`:** + * `max-size-mb`: Set to `-1` to allow the cache to grow dynamically. + * `cache-file-for-range-read`: **Enabled** to avoid downloading entire files for small reads. + * `enable-parallel-downloads`: **Enabled** to potentially improve performance for larger files. +* **`metadata-cache`:** Configured with generous sizes and a 10-minute TTL. +* **`cache-dir`:** Set to `/tmp/cache` (tmpfs). +* **`gcs-connection`:** Increased `max-conns-per-host` and `max-idle-conns-per-host`. +* **`implicit-dirs`:** **Enabled** for faster directory creation. +* **`file-system`:** + * `fuse-options`: Includes `allow_other`, `nonempty`, and `auto_unmount`. + * `temp-dir`: Set to `/tmp/gcsfuse` (tmpfs). + +## Filesystem + +* **Mount point:** `/data` (baced by a hostPath volume, could be extended to use a persistent disk) +* **Cache directory:** `/tmp/cache` (tmpfs) +* **Temporary directory:** `/tmp/gcsfuse` (tmpfs) + +Using tmpfs for the cache and temporary directories leverages node memory for improved performance. However, ensure sufficient memory is available on your nodes. + +## Approach + +The Daemonset uses a privileged container to run GCSFUSE and mount the GCS bucket. A `preStop` lifecycle hook ensures graceful unmounting before the pod terminates. + +## Provide GCS Access to DaemonSet + ```bash + export PROJECT_NUMBER=$(gcloud projects list \ + --filter="PROJECT_ID=${PROJECT_ID}" \ + --format="value(PROJECT_NUMBER)") + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="principal://iam.googleapis.com/projects/${PROJECT_NUMBER}/locations/global/workloadIdentityPools/$PROJECT_ID.svc.id.goog/subject/ns/gcs-node-mount/sa/gcsfuse-node-sa" \ + --role="roles/storage.admin" +``` + +## Build Images and Deploy DaemonSet + +**1. Export Variables** + +```bash +export REGION= +export PROJECT_ID= +``` + +**2. Build Images** +#### Cloudbuild + + Cloudbuild builds the Image from the Dockerfile in the [gcsfuse](https://github.com/GoogleCloudPlatform/gcsfuse/tree/master) Github repo. +```bash +gcloud builds submit --no-source --project $PROJECT_ID --region=$REGION . +``` + +#### Manually + +```bash +git pull https://github.com/GoogleCloudPlatform/gcsfuse.git +cd gcsfuse +docker buildx build --platform linux/amd64 -t $REGION-docker.pkg.dev/$PROJECT_ID/research-images/gcs-fuse:latest --push . +``` + +**3. Deploy Daemonset** + +```bash +kubectl apply -k k8s/ +``` + +**4. Deploy Sample Workload** + +```bash +kubectl apply -f k8s/example-pod.yaml +``` + + Exec into pod and verify that you can see the files in your gcs bucket in the pod. + +```bash +kubectl exec -n default -it pod/my-pod -- sh +ls -la /data +``` + +## Monitoring + +The configuration exports GCSFUSE metrics to Cloud Monitoring. Monitor these metrics to assess performance and identify potential issues. + +## Notes + +* This setup uses `allow_other`, which has security implications. Ensure your GCS bucket has appropriate ACLs and your nodes are secured. +* GCSFUSE provides eventual consistency. If your application requires strong consistency, consider alternative approaches. +* This README provides a high-level overview. Refer to the GCSFUSE documentation for detailed information. + - https://cloud.google.com/storage/docs/gcs-fuse + - https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/docs diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/cloudbuild.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/cloudbuild.yaml new file mode 100644 index 000000000..e67560327 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/cloudbuild.yaml @@ -0,0 +1,26 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- name: 'gcr.io/cloud-builders/git' + args: ['clone', 'https://github.com/GoogleCloudPlatform/gcsfuse.git'] +- name: 'gcr.io/cloud-builders/docker' + dir: 'gcsfuse' + args: ['build', '-t', '$LOCATION-docker.pkg.dev/$PROJECT_ID/research-images/gcsfuse:latest', '.'] +- name: 'gcr.io/cloud-builders/docker' + args: ['push', '$LOCATION-docker.pkg.dev/$PROJECT_ID/research-images/gcsfuse:latest'] +options: + machineType: 'E2_HIGHCPU_8' +images: +- '$LOCATION-docker.pkg.dev/$PROJECT_ID/research-images/gcsfuse:latest' diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/daemonset.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/daemonset.yaml new file mode 100644 index 000000000..9dc95a913 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/daemonset.yaml @@ -0,0 +1,137 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: gcsfuse-config + namespace: gcs-node-mount +data: + gcs_config.yaml: |- + + app-name: "fuse-node-mount" + write: + create-empty-file: true + logging: + format: json + severity: INFO + file-cache: + max-size-mb: -1 # grow until it occupies all the available capacity in your cache directory. + cache-file-for-range-read: true # Helpful for small files, avoids full downloads + enable-parallel-downloads: true # Helps if SOME files are larger + parallel-downloads-per-file: 8 # Tune based on network/GCS limits + max-parallel-downloads: 64 # Node-wide limit, balance concurrency + download-chunk-size-mb: 8 # Align with GCS optimal chunk size + metadata-cache: + enable-nonexistent-type-cache: true # Reduces 'stat' calls for missing files + stat-cache-max-size-mb: 32 + ttl-secs: 600 # 10 mins balance freshess vs cache hits (Tune depending on how often files change) + type-cache-max-size-mb: 4 + cache-dir: "/tmp/cache" + # only-dir: "# Only needed if restricting access to a subpath + gcs-connection: + max-conns-per-host: 128 # High, to handle many pods concurrently + max-idle-conns-per-host: 64 # Keep some alive to reduce connection overhead + sequential-read-size-mb: 32 # Good for sequential reads if they occur + implicit-dirs: true # Helps with 'mkdir -p', but has consistency caveats + file-system: + kernel-list-cache-ttl-secs: 60 # Cache dir listings moderately + ignore-interrupts: true # For better stability under load + dir-mode: "755" # Permissions, adjust as needed + file-mode: "644" # Same as above + fuse-options: "allow_other,nonempty,auto_unmount" # Add specific FUSE options if required + # gid: # If needed for ownership mapping + # uid: # If needed for ownership mapping + # rename-dir-limit: # Tune based on rename behaviour + temp-dir: "/tmp/gcsfuse" # Mapped to tmpfs + foreground: true + gcs-retries: + max-retry-sleep: 30 # Defaults + multiplier: "2" # Defaults + metrics: + stackdriver-export-interval: 60 # Export metrics every 60 seconds for performance monitoring + debug: + log-mutex: false # Only enable if debugging locking issues +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gcsfuse-node-sa + namespace: gcs-node-mount +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gcs-fuse-daemonset + namespace: gcs-node-mount +spec: + selector: + matchLabels: + app: gcs-fuse + template: + metadata: + labels: + app: gcs-fuse + spec: + terminationGracePeriodSeconds: 60 # Time to tidy up mounts + containers: + - name: gcs-fuse + lifecycle: + preStop: + exec: + command: ["fusermount", "-u", "/data"] + image: us-central1-docker.pkg.dev/fsi-research-5/research-images/gcsfuse + command: ["gcsfuse"] + args: + - "-o" + - "allow_other" + - "--config-file" + - "/config/gcs_config.yaml" + - "fuse-mount-testing" + - "/data" + imagePullPolicy: Always + resources: + requests: + cpu: "500m" + memory: "512Mi" + volumeMounts: + - name: gcsfuse-config + mountPath: /config + - name: gcsfuse-storage + mountPath: /data + mountPropagation: Bidirectional + - name: gcsfuse-cache + mountPath: /tmp/cache + - name: tmp-dir + mountPath: /tmp/gcsfuse + - name: fuse-device + mountPath: /dev/fuse + securityContext: + privileged: true + volumes: + - name: gcsfuse-config + configMap: + name: gcsfuse-config + - name: gcsfuse-cache + emptyDir: {} + - name: tmp-dir + emptyDir: {} + - name: fuse-device + hostPath: + path: /dev/fuse + - name: gcsfuse-storage + hostPath: + path: /mnt/disks/gcsfuse-storage + type: DirectoryOrCreate + serviceAccountName: gcsfuse-node-sa diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/example-pod.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/example-pod.yaml new file mode 100644 index 000000000..1360470d3 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/example-pod.yaml @@ -0,0 +1,38 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: busybox + image: busybox + command: ['sh', '-c', 'while true; do sleep 30; done;'] + securityContext: + privileged: true + volumeMounts: + - name: host-path + mountPath: /data + resources: + requests: + cpu: "1" + memory: "2Gi" + restartPolicy: Always + volumes: + - name: host-path + hostPath: + path: /mnt/disks/gcsfuse-storage + type: Directory diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/kustomization.yaml new file mode 100644 index 000000000..8a5ea1d65 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/kustomization.yaml @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- daemonset.yaml +- namespace.yaml +images: +- name: gcsfuse + newName: us-central1-docker.pkg.dev/fsi-research-5/research-images/gcsfuse # Replace with your desired registry + newTag: latest # Replace with your desired tag diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/namespace.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/namespace.yaml new file mode 100644 index 000000000..5615d69b2 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/gcsfuse-nodemount/k8s/namespace.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: gcs-node-mount diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/.Dockerignore b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/.Dockerignore new file mode 100644 index 000000000..85efb008d --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/.Dockerignore @@ -0,0 +1,3 @@ +.env +Dockerfile +requirements.in diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/Dockerfile b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/Dockerfile new file mode 100644 index 000000000..bf091455e --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/Dockerfile @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:12-slim AS build +RUN apt-get update && \ + apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \ + python3 -m venv /venv && \ + /venv/bin/pip install --upgrade pip setuptools wheel + +# Build the virtualenv as a separate step: Only re-execute this step when requirements.txt changes +FROM build AS build-venv +COPY requirements.txt /requirements.txt +RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.txt + +# Copy the virtualenv into a distroless image +FROM gcr.io/distroless/python3-debian12 +COPY --from=build-venv /venv /venv +COPY . /app +WORKDIR /app +ENTRYPOINT ["/venv/bin/python3", "main.py"] diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/README.md b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/README.md new file mode 100644 index 000000000..205db1a75 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/README.md @@ -0,0 +1,111 @@ +# Parallelstore Data Transfer Tool + +This tool allows you to import and export data between Google Cloud Parallelstore and Google Cloud Storage (GCS). It provides a command-line interface to initiate data transfers and monitor their progress. + +Along with a sample Kubernetes Job manifest to run in your kubernetes cluster. + +## Features + +* **Import data from GCS to Parallelstore:** Transfer data from a GCS bucket to a Parallelstore instance. +* **Export data from Parallelstore to GCS:** Transfer data from a Parallelstore instance to a GCS bucket. + +## Prerequisites + +* **Google Cloud Project:** A Google Cloud project with billing enabled. +* **Parallelstore Instance:** A Parallelstore instance in your project. +* **GCS Bucket:** A GCS bucket to store or retrieve data. +* **Service Account:** A service account with permissions to access Parallelstore and GCS. +* **Google Cloud SDK:** Installed and configured with your project. + +## Run via CLI + 1. Install Requirements + ```bash + pip install -r requirements.txt + ``` + Sample Import Job: + ``` + python3 main.py --mode import --gcsbucket my-bucket-name --instance my-instance-name --location us-central1-a + ``` + Sample Export Job: + ``` + python3 main.py --mode export --gcsbucket my-bucket-name --instance my-instance-name --location us-central1-a + ``` +## Run as a Kubernetes Job + + 1. Export variables + + ```bash + export REGION= + export PROJECT_ID= + export BUCKET_NAME= + ``` + 2. Build and Push docker image + + ```bash + docker build . -t $REGION-docker.pkg.dev/$PROJECT_ID/research-images/parallelstore-transfer + + docker push $REGION-docker.pkg.dev/$PROJECT_ID/research-images/parallelstore-transfer + ``` + 3. Grant IAM Permission for Workload + + The user or service account used for initiating the transfer requires the following permissions: + - `parallelstore.instances.exportData` in order to transfer from Parallelstore to Cloud Storage. + - `parallelstore.instances.importData` in order to transfer to Cloud Storage. + + Both of these permissions are granted with the `roles/parallelstore.admin` role. + + You can create a [custom role](https://cloud.google.com/iam/docs/creating-custom-roles) to grant permissions independently. + + In addition, the Parallelstore service account requires the following permission: + + - `roles/storage.admin` on the Cloud Storage bucket. + + To grant this permission, run the following `gcloud` command: + + > **_NOTE:_** This is assuming you are running the job in the default namespace. + + ```bash + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="principal://iam.googleapis.com/projects/$PROJECT_ID/locations/global/workloadIdentityPools/$PROJECT_ID.svc.id.goog/subject/ns/default/sa/parallelstore-data-transfer" \ + --role="roles/storage.admin" + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="principal://iam.googleapis.com/projects/$PROJECT_ID/locations/global/workloadIdentityPools/$PROJECT_ID.svc.id.goog/subject/ns/default/sa/parallelstore-data-transfer" \ + --role="roles/parallelstore.admin" + + ``` + + 4. Configure Job + + In the `k8s/kustomization.yaml` file, update the following parameters with your specifics, including if you want to run an import or export job. + + ``` + args: + - --mode + - import + - --gcsbucket + - + - --instance + - + - --location + - + - --project-id + - + + ``` + + + 5. Run Job + + ```bash + kustomize build k8s/ | kubectl deploy -f - + ``` + 6. Check status of Job + To check the status of the job you can run the following command: + ```bash + kubectl describe job parallelstore-data-transfer + ``` + otherwise if you'd like to see the logs from the job run the following: + ```bash + kubectl logs job/parallelstore-data-transfer + ``` diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/kustomization.yaml new file mode 100644 index 000000000..82eda8628 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/kustomization.yaml @@ -0,0 +1,48 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +metadata: + name: parallelstore-data-transfer +namespace: default +resources: +- sample-job.yaml +- serviceaccount.yaml +images: +- name: parallelstore-transfer + newName: us-central1-docker.pkg.dev/fsi-research-5/research-images/parallelstore-transfer # Replace with your desired registry + newTag: latest123 # Replace with your desired tag +patches: +- patch: |- + apiVersion: batch/v1 + kind: Job + metadata: + name: parallelstore-data-transfer + spec: + template: + spec: + containers: + - name: data-transfer + args: + - --mode + - import + - --gcsbucket + - + - --instance + - + - --location + - + - --project-id + - diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/sample-job.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/sample-job.yaml new file mode 100644 index 000000000..286169023 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/sample-job.yaml @@ -0,0 +1,37 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: parallelstore-data-transfer +spec: + template: + spec: + serviceAccountName: parallelstore-data-transfer + containers: + - name: data-transfer + image: parallelstore-transfer:latest + args: + - --mode + - import + - --gcsbucket + - pstore_export_test + - --instance + - daos-instance + - --location + - us-central1-a + - --project-id + - fsi-research-5 + restartPolicy: Never diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/serviceaccount.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/serviceaccount.yaml new file mode 100644 index 000000000..e664bbf04 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/k8s/serviceaccount.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: parallelstore-data-transfer diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/main.py b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/main.py new file mode 100644 index 000000000..2b81518af --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/main.py @@ -0,0 +1,183 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# flake8: noqa + +from google.cloud import parallelstore_v1 +import logging +import os +import sys +import argparse +import time + +# Configure logging +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +logger = logging.getLogger(__name__) + + +def import_data_to_parallelstore( + gcs_bucket, path, instance, location, request_id, project_id +): + """Imports data from GCS to Parallelstore.""" + + client = parallelstore_v1.ParallelstoreClient() + + source_gcs_bucket = parallelstore_v1.SourceGcsBucket() + source_gcs_bucket.uri = f"gs://{gcs_bucket}" + + request = parallelstore_v1.ImportDataRequest( + source_gcs_bucket=source_gcs_bucket, + name=f"projects/{project_id}/locations/{location}/instances/{instance}", + ) + if request_id is not None: + request.request_id = request_id + if path != "/": + request.destination_parallelstore = path + + operation = client.import_data(request=request) + log_operation_start( + "Import", gcs_bucket, instance, location, project_id, request_id + ) + + wait_for_operation(operation) + + if operation.done(): + response = operation.result() + logger.info(f"Operation succeeded: {response}") + + +def export_data_to_gcs(gcs_bucket, path, instance, location, request_id, project_id): + """Exports data from Parallelstore to GCS.""" + + client = parallelstore_v1.ParallelstoreClient() + + destination_gcs_bucket = parallelstore_v1.DestinationGcsBucket() + destination_gcs_bucket.uri = f"gs://{gcs_bucket}" + + request = parallelstore_v1.ExportDataRequest( + name=f"projects/{project_id}/locations/{location}/instances/{instance}", + destination_gcs_bucket=destination_gcs_bucket, + ) + + if request_id is not None: + request.request_id = request_id + if path != "/": + request.destination_parallelstore = path + + operation = client.export_data(request=request) + log_operation_start( + "Export", gcs_bucket, instance, location, project_id, request_id + ) + + wait_for_operation(operation) + + if operation.done(): + response = operation.result() + logger.info(f"Operation succeeded: {response}") + + +def log_operation_start( + operation_type, gcs_bucket, instance, location, project_id, request_id +): + """Logs the start of an import/export operation.""" + logger.info(f"Starting {operation_type}") + logger.info(f"GCS Bucket: {gcs_bucket}") + logger.info(f"Parallelstore Instance: {instance}") + logger.info(f"Location: {location}") + logger.info(f"Project ID: {project_id}") + logger.info(f"Request ID: {request_id}") + + +def wait_for_operation(operation): + """Waits for a long-running operation to complete.""" + while not operation.done(): + logger.info("Waiting for operation to complete...") + time.sleep(10) + + +def main(): + parser = argparse.ArgumentParser( + description="Import or Export data between Parallelstore and GCS", + epilog="Example usage: python import_data.py --mode import --gcsbucket my-bucket-name --instance my-instance-name --location us-central1-a", + ) + parser.add_argument( + "--mode", + required=True, + help="Import or Export data to / from a Parallelstore Instance", + ) + parser.add_argument( + "--gcsbucket", + required=False, + help="specifies the URI to a Cloud Storage bucket, or a path within a bucket, using the format gs:///", + ) + parser.add_argument( + "--path", + required=False, + default="/", + help="Root directory path to the Parallelstore file system", + ) + parser.add_argument( + "--instance", required=False, help="Parallelstore instance name" + ) + parser.add_argument( + "--location", + required=False, + help="Parallelstore location, must be a supported zone.", + ) + parser.add_argument("--project-id", required=False, help="Project ID") + parser.add_argument( + "--request-id", + required=False, + default=None, + help="allows you to assign a unique ID to this request. If you retry this request using the same request ID, the server will ignore the request if it has already been completed. Must be a valid UUID that is not all zeros.", + ) + args = parser.parse_args() + + # Get values from environment variables or command-line arguments + gcs_bucket = os.environ.get("GCS_BUCKET", args.gcsbucket) + path = os.environ.get("PARALLELSTORE_PATH", args.path) + instance = os.environ.get("PARALLELSTORE_INSTANCE", args.instance) + location = os.environ.get("PARALLELSTORE_LOCATION", args.location) + project_id = os.environ.get("PROJECT_ID", args.project_id) + request_id = os.environ.get("REQUEST_ID", args.request_id) + + # Check for missing values + missing_args = [] + for arg_name, arg_value in [ + ("GCS_BUCKET", gcs_bucket), + ("PARALLELSTORE_INSTANCE", instance), + ("PARALLELSTORE_LOCATION", location), + ("project_id", project_id), + ]: + if arg_value is None: + missing_args.append(arg_name) + + if missing_args: + logger.error(f"Error: Missing required arguments: {', '.join(missing_args)}") + logger.error( + "Please provide them through command-line arguments or environment variables." + ) + exit(1) # Exit with an error code + + if args.mode == "import": + import_data_to_parallelstore( + gcs_bucket, path, instance, location, request_id, project_id + ) + elif args.mode == "export": + export_data_to_gcs(gcs_bucket, path, instance, location, request_id, project_id) + else: + logger.error("Missing operation mode") + + +if __name__ == "__main__": + main() diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/requirements.in b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/requirements.in new file mode 100644 index 000000000..83527a17a --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/requirements.in @@ -0,0 +1 @@ +google-cloud-parallelstore diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/requirements.txt b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/requirements.txt new file mode 100644 index 000000000..f731a4d1e --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore-transfer-tool/requirements.txt @@ -0,0 +1,51 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in +cachetools==5.5.2 + # via google-auth +certifi==2025.1.31 + # via requests +charset-normalizer==3.4.1 + # via requests +google-api-core==2.24.1 + # via google-cloud-parallelstore +google-auth==2.38.0 + # via + # google-api-core + # google-cloud-parallelstore +google-cloud-parallelstore==0.2.11 + # via -r requirements.in +googleapis-common-protos==1.69.1 + # via + # google-api-core + # grpcio-status +grpcio==1.70.0 + # via + # google-api-core + # grpcio-status +grpcio-status==1.70.0 + # via google-api-core +idna==3.10 + # via requests +proto-plus==1.26.0 + # via + # google-api-core + # google-cloud-parallelstore +protobuf==6.30.0 + # via + # google-api-core + # google-cloud-parallelstore + # googleapis-common-protos + # grpcio-status + # proto-plus +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.1 + # via google-auth +requests==2.32.4 + # via google-api-core +rsa==4.9 + # via google-auth +urllib3==2.5.0 + # via requests diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/README.md b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/README.md new file mode 100644 index 000000000..fcab40e84 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/README.md @@ -0,0 +1,124 @@ +# Deploying a DaemonSet for Node Mounting Parallelstore +> **WARNING:** This is not an officially supported Google service. The use of this solution is on an “as-is” basis and is not a service offered under the Google Cloud Terms of Service. + +> **WARNING:** Google does not endorse the PoC approach described in this document for production use. This PoC is intended solely for internal testing to validate that the shared mount architecture for Parallelstore. It should only be used for experimental purposes. Please also note that this PoC does not support Node Autoscaling, non-disruptive node upgrades, or DaemonSet updates. Any attempt at node upgrades or DaemonSet updates will require complete downtime for workloads and will result in workload failures if these actions are attempted. + + +This DaemonSet mounts a Parallelstore instance (Google Cloud's high-performance storage solution) for all Pods on a GKE node to leverage the same mount point. This provides an alternate method to using the supported Parallelstore CSI driver. + +For more information on using the CSI Driver, please refer to the [documentation](https://cloud.google.com/parallelstore/docs/csi-driver-overview). + +## Build Images and Deploy DaemonSet + +**1. Export Variables** + +```bash +export REGION= +export PROJECT_ID= +``` + +**2. Build Images** +#### Cloudbuild +```bash +gcloud builds submit --project $PROJECT_ID --region=$REGION . +``` + +#### Manually +```bash +cd image +docker buildx build --platform linux/amd64 -t $REGION-docker.pkg.dev/$PROJECT_ID/research-images/nodemount:latest --push -f ./Dockerfile . +``` + +**3. Update Config** + +Update the `DAOS_ACCESS_POINTS` with the your Parallelstore Access Points in `k8s/daemonset-access-points.yaml`. + +If you used the included terraform these were outputted as apart of the Terraform Apply. + +Otherwise, you can find these via the following command (replace INSTANCE_NAME and LOCATION with your actual values): + +```bash +gcloud beta parallelstore instances describe INSTANCE_NAME --location=LOCATION +``` +Example output +```bash +accessPoints: +- 10.155.200.4 +- 10.155.200.3 +- 10.155.200.2 +capacityGib: '12000' +createTime: '2024-10-03T14:22:45.381421273Z' +effectiveReservedIpRange: address +name: projects/project-id/locations/us-central1-a/instances/daos-instance +network: projects/project-id/global/networks/research-vpc +state: ACTIVE +updateTime: '2024-10-03T14:31:53.129533289Z' +``` +In k8s/daemonset-access-points.yaml, replace the placeholder access points with your own: + +```yaml +# k8s/daemonset-access-points.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: daos-client +spec: + template: + spec: + containers: + - name: daos-client + env: + - name: DAOS_ACCESS_POINTS + value: "10.90.188.4, 10.90.188.2, 10.90.188.3" # Replace with your access points +``` + +Update `k8s/kustomization.yaml` with your Artifact Registry details. +```yaml +# k8s/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- daemonset.yaml +- daemonset-access-points.yaml +images: +- name: nodemount + newName: us-central1-docker.pkg.dev/your-project-id/research-images/nodemount # Replace with your registry + newTag: latest # Replace with your desired tag +``` + +**4. Apply the DaemonSet to your cluster** + +```bash +kubectl apply -k k8s +``` + +**5. Validate Deployment** +```bash +kubectl describe daemonset/daos-client -n default +``` +Check the "Desired Number Scheduled" and "Current Number Scheduled" to ensure they match the number of nodes in your cluster. +Look for any error messages or events that indicate problems with the DaemonSet. +**6. Test with sample workload** +This example workload deploys two pods that mount the Parallelstore volume. +```bash +kubectl apply -f example-workload.yaml +``` +Exec into the first pod and create a file: +```bash +kubectl exec -it pod/pstore-testing-1 -- sh +touch /data/test.txt +``` +Exec into the second pod and list the directory to see the file created by the first pod: +```bash +kubectl exec -it pod/pstore-testing-2 -- sh +ls -la /data +``` +**7. (Optional) Transfer data to or from Cloud Storage** + +Another option to validate the Daemonset is by using the import data feature of Parallelstore. + +Parallelstore can import data from, and export data to, Cloud Storage. Data transfers allow you to quickly load data into your Parallelstore instance, and to use Cloud Storage as a durable backing layer for your Parallelstore instance. + +Transfers from Cloud Storage are incremental; they only copy files to your Parallelstore instance that don't already exist on the instance, or that have changed since they were transferred. + +Follow guide [here](https://cloud.google.com/parallelstore/docs/transfer-data) to import data into the Parallelstore instance and validate that is visible from one of the pods above. diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/cloudbuild.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/cloudbuild.yaml new file mode 100644 index 000000000..1b2133fa5 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/cloudbuild.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- name: 'gcr.io/cloud-builders/docker' + args: [ 'buildx', 'build', '--platform', 'linux/amd64', '--cache-from', 'type=registry,ref=$LOCATION-docker.pkg.dev/$PROJECT_ID/research-images/node-mount', '-t', '$LOCATION-docker.pkg.dev/$PROJECT_ID/research-images/image:latest', '-f', 'node-mount/Dockerfile', '--push','.' ] + dir: 'image/' diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/example-workload.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/example-workload.yaml new file mode 100644 index 000000000..d01d33ff8 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/example-workload.yaml @@ -0,0 +1,57 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: pstore-testing-1 +spec: + containers: + - name: pstore-testing + image: busybox + command: ['sh', '-c', 'while true; do sleep 30; done;'] + volumeMounts: + - name: parallelstore-mount + mountPath: /data + resources: + requests: + cpu: "1" + memory: "2Gi" + restartPolicy: Always + volumes: + - name: parallelstore-mount + hostPath: + path: /tmp/parallelstore +--- +apiVersion: v1 +kind: Pod +metadata: + name: pstore-testing-2 +spec: + containers: + - name: pstore-testing + image: busybox + command: ['sh', '-c', 'while true; do sleep 30; done;'] + volumeMounts: + - name: parallelstore-mount + mountPath: /data + resources: + requests: + cpu: "1" + memory: "2Gi" + restartPolicy: Always + volumes: + - name: parallelstore-mount + hostPath: + path: /tmp/parallelstore diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/Dockerfile b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/Dockerfile new file mode 100644 index 000000000..6884adb13 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/Dockerfile @@ -0,0 +1,159 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# FROM debian:12-slim AS build + +# RUN apt-get update && \ +# apt-get install -y \ +# curl \ +# gnupg2 \ +# libpci3 \ +# apt-transport-https \ +# dirmngr \ +# supervisor \ +# gettext-base && \ +# curl https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg | apt-key add - && \ +# echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" > /etc/apt/sources.list.d/artifact-registry.list && \ +# apt-get update && \ +# apt-get install -y daos-client && \ +# rm -rf /var/lib/apt/lists/* && \ +# mkdir -p /var/run/daos_agent /tmp/parallelstore + +# FROM debian:12-slim + +# ENV LIB_DIR_PREFIX=x86_64 \ +# LD_LINUX_FILE=/lib64/ld-linux-x86-64.so.2 \ +# LIB_DIR=/lib64/ \ +# LD_LIBRARY_PATH=/lib64 + +# COPY --from=build \ +# /bin/mount \ +# /bin/umount \ +# /bin/sh \ +# /bin/ + +# COPY --from=build \ +# /usr/bin/supervisord \ +# /usr/bin/supervisorctl \ +# /usr/bin/envsubst \ +# /usr/local/bin/ + +# COPY --from=build /etc/supervisor/ /etc/supervisor/ +# COPY --from=build ${LD_LINUX_FILE} ${LIB_DIR} +# COPY --from=build /lib/lib*.so* /lib/ +# COPY --from=build /lib64/lib*.so* /lib64/ +# COPY --from=build /lib/${LIB_DIR_PREFIX}-linux-gnu/lib*.so* /lib/${LIB_DIR_PREFIX}-linux-gnu/ +# COPY --from=build \ +# /usr/bin/daos_agent \ +# /usr/bin/dfuse \ +# /usr/bin/ + +# COPY --from=build /etc/daos /etc/daos/ +# COPY --from=build /var/run/daos_agent /var/run/daos_agent/ +# COPY --from=build /tmp/parallelstore /tmp/parallelstore/ + +# RUN echo '[supervisord]\n\ +# nodaemon=true\n\ +# \n\ +# [program:daos_agent]\n\ +# command=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml\n\ +# \n\ +# [program:dfuse]\n\ +# command=/usr/bin/dfuse %(ENV_DFUSE_ARGS)s\n\ +# depends_on=daos_agent' > /etc/supervisord-template.conf + +# RUN echo '#!/bin/sh\n\ +# if [ -z "$DFUSE_ARGS" ]; then\n\ +# export DFUSE_ARGS="-f -m=/mnt/daos --thread-count=32 --eq-count=16 --pool=default-pool --container=default-container --disable-wb-cache --multi-user"\n\ +# fi\n\ +# envsubst < /etc/supervisord-template.conf > /etc/supervisor.conf\n\ +# exec /usr/local/bin/supervisord -c /etc/supervisor.conf' > /generate-config.sh && \ +# chmod +x /generate-config.sh + +# EXPOSE 10001 + +# ENTRYPOINT ["/generate-config.sh"] + +#### +# Build stage for supervisord +FROM golang:alpine AS supervisor-builder + +RUN apk add --no-cache --update git gcc rust + +WORKDIR /tmp +RUN git clone https://github.com/ochinchina/supervisord.git +WORKDIR /tmp/supervisord/ + +RUN go build -a -ldflags "-linkmode external -extldflags -static" -o /usr/local/bin/supervisord github.com/ochinchina/supervisord + +# RUN CGO_ENABLED=0 go build -a -ldflags "-linkmode external -extldflags -static" -o /usr/local/bin/supervisord github.com/ochinchina/supervisord + + +# Build stage for DAOS +FROM debian:12-slim AS build +RUN apt-get update && \ + apt-get install -y \ + curl gnupg2 libpci3 apt-transport-https dirmngr gettext-base && \ + curl https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg | apt-key add - && \ + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" > /etc/apt/sources.list.d/artifact-registry.list && \ + apt-get update && \ + apt-get install -y daos-client && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p /var/run/daos_agent /tmp/parallelstore + +# Final stage +FROM gcr.io/distroless/base-debian12 + +ENV LIB_DIR_PREFIX=x86_64 \ + LD_LINUX_FILE=/lib64/ld-linux-x86-64.so.2 \ + LIB_DIR=/lib64/ \ + LD_LIBRARY_PATH=/lib64 + +# Copy static supervisord +COPY --from=supervisor-builder /usr/local/bin/supervisord /usr/bin/supervisord + +# Copy required binaries and libraries +COPY --from=build \ + /usr/bin/envsubst \ + /usr/bin/daos_agent \ + /usr/bin/dfuse \ + /usr/bin/ + +COPY --from=build \ + /bin/mount \ + /bin/umount \ + /bin/sh \ + /bin/chmod \ + /bin/ + +# Copy libraries +COPY --from=build ${LD_LINUX_FILE} ${LIB_DIR} +COPY --from=build /lib/lib*.so* /lib/ +COPY --from=build /lib64/lib*.so* /lib64/ +COPY --from=build /lib/${LIB_DIR_PREFIX}-linux-gnu/lib*.so* /lib/${LIB_DIR_PREFIX}-linux-gnu/ + +# Copy DAOS configurations +COPY --from=build /etc/daos /etc/daos/ +COPY --from=build /var/run/daos_agent /var/run/daos_agent/ +COPY --from=build /tmp/parallelstore /tmp/parallelstore/ + +# Create supervisor config template +COPY generate-config.sh /generate-config.sh +COPY supervisord-template.conf /etc/supervisord-template.conf + +RUN chmod +x /generate-config.sh + +EXPOSE 10001 + +ENTRYPOINT ["/generate-config.sh"] diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/generate-config.sh b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/generate-config.sh new file mode 100644 index 000000000..8b0800167 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/generate-config.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set default DFUSE_ARGS if not provided +if [ -z "$DFUSE_ARGS" ]; then + export DFUSE_ARGS="-f -m=/mnt/daos --thread-count=32 --eq-count=16 --pool=default-pool --container=default-container --disable-wb-cache --multi-user" +fi + +# Generate final supervisor config +envsubst < /etc/supervisord-template.conf > /etc/supervisor.conf + +# Start supervisor with generated config +exec /usr/bin/supervisord -c /etc/supervisor.conf diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/supervisord-template.conf b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/supervisord-template.conf new file mode 100644 index 000000000..9bd1ef5e8 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/image/supervisord-template.conf @@ -0,0 +1,9 @@ +[supervisord] +nodaemon=true + +[program:daos_agent] +command=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + +[program:dfuse] +command=/usr/bin/dfuse %(ENV_DFUSE_ARGS)s +depends_on=daos_agent diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/daemonset-access-points.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/daemonset-access-points.yaml new file mode 100644 index 000000000..e7309db16 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/daemonset-access-points.yaml @@ -0,0 +1,26 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: parallelstore-nodemount +spec: + template: + spec: + containers: + - name: parallelstore-nodemount + env: + - name: DAOS_ACCESS_POINTS + value: "10.155.200.4, 10.155.200.2, 10.155.200.3" # Replace with your access points diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/daemonset.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/daemonset.yaml new file mode 100644 index 000000000..35a89f57c --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/daemonset.yaml @@ -0,0 +1,107 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: parallelstore-nodemount +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: parallelstore-nodemount + namespace: parallelstore-nodemount +data: + daos_agent.yml: |- + access_points: ['10.93.0.3', '10.93.0.4', '10.93.0.2'] # change to parallelstore server IP address + + credential_config: + client_user_map: + default: + user: nobody + group: nobody + + transport_config: + allow_insecure: true + + log_file: /daos-tmp/daos_agent.log + + fabric_ifaces: + - numa_node: 0 + devices: + - iface: eth0 + domain: eth0 +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: parallelstore-nodemount + namespace: parallelstore-nodemount +spec: + selector: + matchLabels: + app: parallelstore-nodemount + template: + metadata: + labels: + app: parallelstore-nodemount + spec: + containers: + - name: parallelstore-nodemount + image: nodemount # Replace with your image reference + imagePullPolicy: Always + securityContext: + privileged: true + volumeMounts: + - name: daos-agent + mountPath: /etc/daos + - name: parallelstore-mount + mountPath: /mnt/daos + mountPropagation: Bidirectional + - name: daos-agent-socket + mountPath: /var/run/daos_agent + - name: daos-tmp + mountPath: /daos-tmp + - name: host-sysfs + mountPath: /sys + ports: + - containerPort: 10001 + env: + - name: D_LOG_MASK + value: INFO + - name: DFUSE_ARGS + value: "-f -m=/mnt/daos --thread-count=32 --eq-count=16 --pool=default-pool --container=default-container --disable-wb-cache --multi-user" + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + volumes: + - name: daos-agent + configMap: + name: parallelstore-nodemount + - name: daos-agent-socket + emptyDir: {} + - name: daos-tmp + emptyDir: {} + - name: parallelstore-mount + hostPath: + path: /tmp/parallelstore + type: DirectoryOrCreate + - name: host-sysfs + hostPath: + path: /sys + type: Directory diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/kustomization.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/kustomization.yaml new file mode 100644 index 000000000..7b233c489 --- /dev/null +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/parallelstore/k8s/kustomization.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- daemonset.yaml +images: +- name: nodemount + newName: us-docker.pkg.dev/fsi-research-1/research-images/nodemount # Replace with your desired registry + newTag: latest # Replace with your desired tag diff --git a/5-appinfra/modules/htc-infra/loadtest/.gitignore b/5-appinfra/modules/htc-infra/loadtest/.gitignore new file mode 100644 index 000000000..96e4acabc --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/.gitignore @@ -0,0 +1,5 @@ +# Produced binary +loadtest + +# Generated protobuf code +*.pb.go diff --git a/5-appinfra/modules/htc-infra/loadtest/Dockerfile b/5-appinfra/modules/htc-infra/loadtest/Dockerfile new file mode 100644 index 000000000..1501f84c0 --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.24 AS bld + +# Fetch building tools +RUN apt-get update && apt-get install -y \ + ca-certificates openssl \ + golang-google-protobuf-dev \ + protoc-gen-go \ + protoc-gen-go-grpc \ + && rm -rf /var/lib/apt/lists/* + +# Fetch Dependencies +WORKDIR /build/go +COPY go.* /build/go/ +RUN go mod download + +# Copy in code and build +COPY . /build/go/ +RUN go generate +RUN CGO_ENABLED=0 go build + +# Final build +FROM gcr.io/distroless/static-debian12 + +COPY --from=bld /build/go/loadtest /loadtest + +ENTRYPOINT [ "/loadtest" ] diff --git a/5-appinfra/modules/htc-infra/loadtest/README.md b/5-appinfra/modules/htc-infra/loadtest/README.md new file mode 100644 index 000000000..140abbc24 --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/README.md @@ -0,0 +1,99 @@ +# Load Test for HTC + +## Overview + +This is a general purpose gRPC load test. It supports compute, reading/writing data, +initialization, and produces JSON formatted stats. + +## Compilation + +This can be compiled using `go generate` and `go build`. The following packages +are required for building: `protoc-gen-go`, `protoc-gen-go-grpc`, and +`golang-google-protobuf-dev`. There is a provided [Dockerfile](Dockerfile). + +## Running locally + +### Building + +```sh +go generate +go build +``` + +### Generate test data + +Generate 1000 example tasks: + +```sh +./loadtest gentasks --count 1000 tasks.jsonl +``` + +### With the file directly + +Directly run loadtest with the test data: + +```sh +./loadtest load tasks.jsonl +``` + +You will see both logging (on stderr) and the generated output (on stdout). + +### With gRPC + +Run the pricing engine locally on the default port 2002: + +```sh +./loadtest serve +``` + +In a separate command prompt, [grpcurl](https://github.com/fullstorydev/grpcurl) with the test data to try it out: +to generate JSON test data for the pricing library. This will only send in the +first request. + +```sh +head -n 1 tasks.jsonl | grpcurl -d @ -plaintext localhost:2002 main.PricingService/CalcPrices +``` + +NOTE: You can use the --logJSON to produce JSON formatted logs which includes some +performance statistics. These can be directly queried with distributed jobs using +Google Cloud Logging with [Log Analytics](https://cloud.google.com/logging/docs/analyze/query-and-view). + +## Running in a container + +### Build the container + +```sh +docker build -t loadtest . +``` + +### Generate test data + +Generate 1000 example tasks: + +```sh +docker run loadtest gentasks --count 1000 - > tasks.jsonl +``` + +### With the file directly + +Directly run loadtest with the test data: + +```sh +docker run -v $PWD:/data loadtest load /data/tasks.jsonl +``` + +### With gRPC + +Run the loadtest engine locally on the default port 2002: + +```sh +docker run -p 2002:2002 loadtest serve +``` + +In a separate command prompt, [grpcurl](https://github.com/fullstorydev/grpcurl) with the test data to try it out: +to generate JSON test data for the pricing library. This will only send in the +first request. + +```sh +head -n 1 tasks.jsonl | grpcurl -d @ -plaintext localhost:2002 main.LoadTestService/RunLibrary +``` diff --git a/5-appinfra/modules/htc-infra/loadtest/generate.go b/5-appinfra/modules/htc-infra/loadtest/generate.go new file mode 100644 index 000000000..f0fec377c --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/generate.go @@ -0,0 +1,221 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + crand "crypto/rand" + "fmt" + "log/slog" + "math/rand" + "slices" + + "github.com/spf13/cobra" + "google.golang.org/protobuf/encoding/protojson" +) + +/* + * Add the command line for reading data in parallel + */ + +func AddReadDataCommand(rootCmd *cobra.Command) { + parallel := 20 + progress := true + readData := &cobra.Command{ + Use: "readdata

", + Short: "Read data recursively in a folder with parallel readers", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + + // Find all files + files := make([]string, 0, 100) + var totalBytes int64 + err := WalkDirFiles(args[0], true, func(path string, size int64) error { + files = append(files, path) + totalBytes += size + return nil + }) + if err != nil { + return err + } + + // Info + slog.Info("Reading data", "dir", args[0], "files", len(files), "bytes", totalBytes, "readers", parallel) + + // Suppress progress if requested + if !progress { + totalBytes = 0 + } + + return ApplyParallelWithStats(slices.Values(files), parallel, totalBytes, ReadBytes) + }, + } + readData.Flags().BoolVar(&progress, "showProgress", progress, "Show progress") + readData.Flags().IntVar(¶llel, "parallel", parallel, "Number of parallel readers") + rootCmd.AddCommand(readData) +} + +/* + * Add the command line for writing data in parallel + */ + +func AddWriteDataCommand(rootCmd *cobra.Command) { + + // Write data + sizeBytes := int64(1024 * 1024) + count := 100 + parallel := 2 + progress := true + writeData := &cobra.Command{ + Use: "writedata ", + Short: "Write data (file-.bin) in a folder", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + + // Create directory if not existing + slog.Info("Making parent directory", "dir", args[0]) + if err := MkdirAll(args[0]); err != nil { + return err + } + + // File name generator + slog.Info("Writing data", "count", count, "size", sizeBytes, "writers", parallel) + files := func(yield func(string) bool) { + for i := 0; i < count; i++ { + if !yield(Join(args[0], fmt.Sprintf("file-%08d.bin", i+1))) { + break + } + } + } + + // Setup progress if requested + totalProgress := int64(0) + if progress { + totalProgress = int64(count) * sizeBytes + } + + return ApplyParallelWithStats(files, parallel, totalProgress, func(file string) (int64, error) { + return WriteBytes(file, sizeBytes) + }) + }, + } + writeData.Flags().BoolVar(&progress, "showProgress", progress, "Show progress") + writeData.Flags().IntVar(¶llel, "parallel", parallel, "Number of parallel writers") + writeData.Flags().IntVar(&count, "count", count, "Number of files to write") + writeData.Flags().Int64Var(&sizeBytes, "size", sizeBytes, "Size of files to write") + rootCmd.AddCommand(writeData) +} + +/* + * Add the command line for generating tasks for load test (JSON formatted) + */ + +func AddGenTasksCommand(rootCmd *cobra.Command) { + + // Generate Tasks file command + percCrash := 0.0 + percFail := 0.0 + initMinMicros := int64(1_000_000) + initMaxMicros := int64(1_000_000) + minMicros := int64(100_000) + maxMicros := int64(100_000) + initReadDir := "" + readDir := "" + readFileDir := "" + writeFileDir := "" + writeBytes := int64(0) + resultSize := int64(100) + payloadSize := int64(100) + count := 0 + genTasks := &cobra.Command{ + Use: "gentasks dir", + Short: "Generate JSON tasks for being distributed to the loadtest gRPC service", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + + initialId := rand.Int31() + + // Generate data and convert to JSON + slog.Info("Generating tasks", "count", count) + data := func(yield func(string) bool) { + for i := 0; i < count; i++ { + buf := make([]byte, payloadSize) + _, err := crand.Read(buf) + if err != nil { + slog.Warn("error generating payload", "error", err) + return + } + + // Calculate the read and write file (if a dir is specified) + readFile := "" + if readFileDir != "" { + readFile = Join(readFileDir, fmt.Sprintf("file-%08d.bin", int64(i+1))) + } + writeFile := "" + if writeFileDir != "" { + writeFile = Join(writeFileDir, fmt.Sprintf("file-%08d.bin", int64(i+1))) + } + + b, err := protojson.Marshal(&LoadTask{ + Init: &LoadTask_Task{ + Id: int64(initialId), + PercCrash: percCrash, + PercFail: percFail, + MaxMicros: initMaxMicros, + MinMicros: initMinMicros, + ReadDir: initReadDir, + }, + Task: &LoadTask_Task{ + Id: int64(i + 1), + PercCrash: percCrash, + PercFail: percFail, + MaxMicros: maxMicros, + MinMicros: minMicros, + ReadDir: readDir, + ReadFile: readFile, + WriteFile: writeFile, + WriteBytes: writeBytes, + ResultSize: resultSize, + Payload: buf, + }, + }) + if err != nil { + slog.Warn("error converting to JSON", "error", err) + return + } + if !yield(string(b)) { + break + } + } + } + + return WriteLines(args[0], data) + }, + } + genTasks.Flags().Float64Var(&percCrash, "percCrash", percCrash, "Percentage likelihood of a crash") + genTasks.Flags().Float64Var(&percFail, "percFail", percFail, "Percentage likelihood of a failure") + genTasks.Flags().Int64Var(&initMinMicros, "initMinMicros", initMinMicros, "Initial minimum microsecond work") + genTasks.Flags().Int64Var(&initMaxMicros, "initMaxMicros", initMaxMicros, "Initial maximum microsecond work") + genTasks.Flags().Int64Var(&minMicros, "minMicros", minMicros, "Minimum microsecond work") + genTasks.Flags().Int64Var(&maxMicros, "maxMicros", maxMicros, "Maximum microsecond work") + genTasks.Flags().Int64Var(&resultSize, "resultSize", resultSize, "Result payload size in bytes") + genTasks.Flags().Int64Var(&payloadSize, "payloadSize", payloadSize, "Payload size in bytes") + genTasks.Flags().StringVar(&initReadDir, "initReadDir", initReadDir, "Initial read directory") + genTasks.Flags().StringVar(&readDir, "readDir", readDir, "Read directory for task") + genTasks.Flags().StringVar(&readFileDir, "readFileDir", readFileDir, "Read file prefix for task") + genTasks.Flags().StringVar(&writeFileDir, "writeFileDir", writeFileDir, "Write file prefix for task") + genTasks.Flags().Int64Var(&writeBytes, "writeBytes", writeBytes, "Write file size") + genTasks.Flags().IntVar(&count, "count", count, "Count of records created") + rootCmd.AddCommand(genTasks) +} diff --git a/5-appinfra/modules/htc-infra/loadtest/go.mod b/5-appinfra/modules/htc-infra/loadtest/go.mod new file mode 100644 index 000000000..9c33ab42a --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/go.mod @@ -0,0 +1,64 @@ +module github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest + +go 1.24 + +require ( + cloud.google.com/go/compute/metadata v0.6.0 + cloud.google.com/go/storage v1.50.0 + github.com/schollz/progressbar/v3 v3.16.1 + github.com/spf13/cobra v1.8.1 + google.golang.org/api v0.215.0 + google.golang.org/grpc v1.68.1 + google.golang.org/protobuf v1.36.4 +) + +require ( + cel.dev/expr v0.19.0 // indirect + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.14.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect + cloud.google.com/go/iam v1.2.2 // indirect + cloud.google.com/go/monitoring v1.21.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.3 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.32.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/sdk v1.34.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/term v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 // indirect + google.golang.org/grpc/stats/opentelemetry v0.0.0-20241028142157-ada6787961b3 // indirect +) diff --git a/5-appinfra/modules/htc-infra/loadtest/go.sum b/5-appinfra/modules/htc-infra/loadtest/go.sum new file mode 100644 index 000000000..15a780e1c --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/go.sum @@ -0,0 +1,232 @@ +cel.dev/expr v0.19.0 h1:lXuo+nDhpyJSpWxpPVi5cPUwzKb+dsdOiw6IreM5yt0= +cel.dev/expr v0.19.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM= +cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A= +cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= +cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= +cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= +cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= +cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= +cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= +cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= +cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU= +cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= +cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= +cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= +cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= +cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1 h1:oTX4vsorBZo/Zdum6OKPA4o7544hm6smoRv1QjpTwGo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.3 h1:hVEaommgvzTjTd4xCaFd+kEQ2iYBtGxP6luyLrx6uOk= +github.com/envoyproxy/go-control-plane/envoy v1.32.3/go.mod h1:F6hWupPfh75TBXGKA++MCT/CZHFq5r9/uwt/kQYkZfE= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.16.1 h1:RnF1neWZFzLCoGx8yp1yF7SDl4AzNDI5y4I0aUJRrZQ= +github.com/schollz/progressbar/v3 v3.16.1/go.mod h1:I2ILR76gz5VXqYMIY/LdLecvMHDPVcQm3W/MSKi1TME= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.32.0 h1:P78qWqkLSShicHmAzfECaTgvslqHxblNE9j62Ws1NK8= +go.opentelemetry.io/contrib/detectors/gcp v1.32.0/go.mod h1:TVqo0Sda4Cv8gCIixd7LuLwW4EylumVWfhjZJjDD4DU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 h1:PS8wXpbyaDJQ2VDHHncMe9Vct0Zn1fEjpsjrLxGJoSc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.215.0 h1:jdYF4qnyczlEz2ReWIsosNLDuzXyvFHJtI5gcr0J7t0= +google.golang.org/api v0.215.0/go.mod h1:fta3CVtuJYOEdugLNWm6WodzOS8KdFckABwN4I40hzY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= +google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 h1:J1H9f+LEdWAfHcez/4cvaVBox7cOYT+IU6rgqj5x++8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= +google.golang.org/grpc/stats/opentelemetry v0.0.0-20241028142157-ada6787961b3 h1:hUfOButuEtpc0UvYiaYRbNwxVYr0mQQOWq6X8beJ9Gc= +google.golang.org/grpc/stats/opentelemetry v0.0.0-20241028142157-ada6787961b3/go.mod h1:jzYlkSMbKypzuu6xoAEijsNVo9ZeDF1u/zCfFgsx7jg= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= +google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/5-appinfra/modules/htc-infra/loadtest/main.go b/5-appinfra/modules/htc-infra/loadtest/main.go new file mode 100644 index 000000000..5bfa2b48f --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/main.go @@ -0,0 +1,204 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log/slog" + "os" + + "context" + "net" + + "cloud.google.com/go/compute/metadata" + "github.com/spf13/cobra" + grpc "google.golang.org/grpc" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/reflection" + "google.golang.org/protobuf/encoding/protojson" +) + +/* + * Generate the request / response proto code and gRPC bindings + */ + +//go:generate protoc -I. --go-grpc_out=. --go-grpc_opt=paths=source_relative service.proto +//go:generate protoc -I. --go_out=. --go_opt=paths=source_relative service.proto request.proto response.proto + +/* + * gRPC Service + */ + +type LoadTestServer struct { + UnimplementedLoadTestServiceServer + + cnt *workCounter + hostname string +} + +func NewLoadTestServer(maxWorkers int, hostname string) *LoadTestServer { + return &LoadTestServer{ + cnt: NewWorkerCounter(maxWorkers), + } +} + +func (q *LoadTestServer) RunLibrary(ctx context.Context, in *LoadTask) (*LoadResult, error) { + + // Limit the number of workers that can run + q.cnt.Acquire() + defer q.cnt.Release() + + return RunLoadTask(in, q.hostname) +} + +/* + * Command line utility + */ + +const LONG_DESCRIPTION = ` +Loadtest is an HTC workload that performs simulated work for test purposes. +` + +func main() { + + // Root command + var debug bool + var logJson bool + var numWorkers int + rootCmd := &cobra.Command{ + Use: "loadtest", + Short: "loadtest is an gRPC test harness for testing HTC compute on GCP", + Long: LONG_DESCRIPTION, + + // Run before all subcommands + PersistentPreRun: func(cmd *cobra.Command, args []string) { + + // Log JSON if requested + if logJson { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) + } + + // Enable debug if requested + if debug { + slog.SetLogLoggerLevel(slog.LevelDebug) + slog.Info("Debug logging enabled") + } + }, + } + rootCmd.PersistentFlags().IntVar(&numWorkers, "workers", 1, "Number of worker threads") + rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Enable debug logging") + rootCmd.PersistentFlags().BoolVar(&logJson, "logJSON", false, "Enable JSON logging") + + // Run GRPC Server file command + port := 2002 + serveCmd := &cobra.Command{ + Use: "serve", + Short: "Run gRPC service for simulating work", + RunE: func(cmd *cobra.Command, args []string) error { + + // Find the hostname + hostname, err := os.Hostname() + if err != nil || hostname == "" || hostname == "localhost" { + hostname, err = metadata.InstanceIDWithContext(context.Background()) + if err != nil { + hostname = "localhost" + } + } + + // Listen to the port on all IP addresses + lis, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port)) + if err != nil { + return fmt.Errorf("failed to listen: %v", err) + } + + // Create health check service. + // This can be used to push back on too many tasks in the future. + healthcheck := health.NewServer() + healthcheck.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING) + + // Create server and register all services: + // core service, health check, reflection + s := grpc.NewServer() + RegisterLoadTestServiceServer(s, NewLoadTestServer(numWorkers, hostname)) + grpc_health_v1.RegisterHealthServer(s, healthcheck) + reflection.Register(s) + + // Start server + slog.Info("Listening", "port", port, "hostname", hostname) + if err := s.Serve(lis); err != nil { + return fmt.Errorf("failed to serve: %v", err) + } + + return nil + }, + } + serveCmd.Flags().IntVar(&port, "port", port, "Port for gRPC server") + rootCmd.AddCommand(serveCmd) + + // Load file command + rootCmd.AddCommand(&cobra.Command{ + Use: "load ", + Short: "Load JSONL files and execute directly", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + for line, err := range ReadLines(args[0]) { + if err != nil { + return err + } + + task := &LoadTask{} + if err := protojson.Unmarshal([]byte(line), task); err != nil { + return fmt.Errorf("parsing json: %w", err) + } + response, err := RunLoadTask(task, "localhost") + if err != nil { + return fmt.Errorf("running quant: %w", err) + } + + // Send output + omsg, err := protojson.Marshal(response) + if err != nil { + return fmt.Errorf("running quant: %w", err) + } + + if _, err := os.Stdout.Write(omsg); err != nil { + fmt.Printf("warning: failed to write to stdout: %v", err) + + } + if _, err := os.Stdout.Write([]byte{'\n'}); err != nil { + fmt.Printf("warning: failed to write newline to stdout: %v", err) + } + } + + return nil + }, + }) + + // Add additional commands + AddGenTasksCommand(rootCmd) + AddReadDataCommand(rootCmd) + AddWriteDataCommand(rootCmd) + + // Execute and shutdown quickly on error + rootCmd.InitDefaultHelpCmd() + rootCmd.InitDefaultHelpFlag() + if err := rootCmd.Execute(); err != nil { + slog.Error("error running", "error", err) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/5-appinfra/modules/htc-infra/loadtest/request.proto b/5-appinfra/modules/htc-infra/loadtest/request.proto new file mode 100644 index 000000000..f97f600fd --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/request.proto @@ -0,0 +1,60 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package main; + +option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; + +message LoadTask { + + // Individual task + message Task { + + // Unique id for this task + int64 id = 1; + + // Likelihood of failure (for a simulator) + double perc_crash = 2; + double perc_fail = 3; + + // How long it takes to run (for compute) + int64 max_micros = 4; + int64 min_micros = 5; + + // Size of payload results (all risk metrics, sensitivities, etc) + int64 result_size = 6; + + // Payload includes all marketdata, trade information, + // risk metric parameters, etc.. + bytes payload = 7; + + // Read from a POSIX file + string read_file = 8; + + // Read from a POSIX dir (all files in the directory) + string read_dir = 9; + + // Write output to a POSIX file + string write_file = 10; + int64 write_bytes = 11; + } + + // Initialization task (do it only once for the same id!) + Task init = 1; + + // This task + Task task = 2; +} diff --git a/5-appinfra/modules/htc-infra/loadtest/response.proto b/5-appinfra/modules/htc-infra/loadtest/response.proto new file mode 100644 index 000000000..5828dd25c --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/response.proto @@ -0,0 +1,42 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package main; + +option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; + +message LoadResult { + + message Status { + int64 id = 1; + bytes payload = 2; + + // Compute stats + int64 compute_micros = 3; + + // IO stats + int64 files_read = 4;; + int64 bytes_read = 5; + int64 bytes_written = 6; + + // Total time taken + int64 total_micros = 7; + } + + Status init = 1; + Status task = 2; + string host = 3; +} diff --git a/5-appinfra/modules/htc-infra/loadtest/service.proto b/5-appinfra/modules/htc-infra/loadtest/service.proto new file mode 100644 index 000000000..e66c04211 --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/service.proto @@ -0,0 +1,28 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package main; + +option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; + +import "request.proto"; +import "response.proto"; + +service LoadTestService { + + // Runs the simulated library + rpc RunLibrary(main.LoadTask) returns (main.LoadResult) {} +} diff --git a/5-appinfra/modules/htc-infra/loadtest/simulator.go b/5-appinfra/modules/htc-infra/loadtest/simulator.go new file mode 100644 index 000000000..4aa34b936 --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/simulator.go @@ -0,0 +1,293 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + crand "crypto/rand" + "fmt" + "hash/crc32" + "log/slog" + "math/rand" + "os" + "path" + "sync" + "time" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +/* + * Utility for turning a protobuf into a JSON using structured logging + */ + +type loggableProto struct { + msg proto.Message +} + +// String() turns it into a String +func (p loggableProto) String() string { + x, err := protojson.Marshal(p.msg) + if err != nil { + return string(x) + } else { + return "" + } +} + +// For slog, MarshalJSON is used to bridge into protobuf's JSON marshaler +func (p loggableProto) MarshalJSON() ([]byte, error) { + return protojson.Marshal(p.msg) +} + +/* + * General simulator code + */ + +// Run Quant Library +func RunLoadTask(task *LoadTask, hostname string) (*LoadResult, error) { + slog.Debug("Running LoadTask", "task", loggableProto{task}) + + // Initialize (if needed) + initStatus, err := initialize(task.Init) + if err != nil { + slog.Debug("Error initializing", "err", err) + return nil, fmt.Errorf("error initializing: %w", err) + } + + // Run work + taskStatus, err := runWork(task.Task, "task") + if err != nil { + slog.Debug("Error performing task", "err", err) + return nil, fmt.Errorf("error performing task: %w", err) + } + + // Assemble response + response := &LoadResult{ + Init: initStatus, + Task: taskStatus, + Host: hostname, + } + + slog.Debug("LoadResult", "result", loggableProto{response}) + + return response, nil +} + +// Mutex for the initialization. +var initialMu sync.Mutex +var initialId int64 = -1 + +// initialize +// +// Run the task during initialization. +func initialize(compute_task *LoadTask_Task) (*LoadResult_Status, error) { + + if compute_task == nil { + return &LoadResult_Status{}, nil + } + + // Only run once per thread (regardless of workers) + initialMu.Lock() + defer initialMu.Unlock() + + // Initialize if needed + if compute_task.Id != initialId { + slog.Info("Initializing", "id", compute_task.Id) + status, err := runWork(compute_task, "init") + if err != nil { + return nil, err + } + + initialId = compute_task.Id + + return status, nil + } + + // Return no stats -- acknowledging already initialized + return &LoadResult_Status{ + Id: compute_task.Id, + }, nil +} + +// runWork +// +// Run the specific task (whether an init or normal task), returning the status and +// any error. +// +// This will include all compute, read, and write tasks. +func runWork(task *LoadTask_Task, worktype string) (*LoadResult_Status, error) { + + slog.Debug("Running QuantTask", "task", loggableProto{task}) + + // If there is no task, empty status is enough + if task == nil { + return &LoadResult_Status{}, nil + } + + // Capture total micros (internal) time spent + startTime := time.Now() + + var err error + var bytesRead int64 = 0 + var filesRead int64 = 0 + + // Read files if needed + if task.GetReadDir() != "" { + dirFilesRead, dirBytesRead, err := ReadBytesFromDir(task.GetReadDir()) + if err != nil { + return nil, err + } + bytesRead += dirBytesRead + filesRead += dirFilesRead + } + + // Read file if needed + if task.GetReadFile() != "" { + fileBytesRead, err := ReadBytes(task.GetReadFile()) + if err != nil { + return nil, err + } + bytesRead += fileBytesRead + filesRead += 1 + } + + // Simulate work + computeMicros, status := simulateWork( + task.GetPercCrash(), + task.GetPercFail(), + task.GetMinMicros(), + task.GetMaxMicros()) + + // Write file if needed + var bytesWritten int64 = 0 + if task.GetWriteFile() != "" { + + // Create directory if not existing + if err := os.MkdirAll(path.Dir(task.GetWriteFile()), 0750); err != nil { + return nil, err + } + + // Write file + bytesWritten, err = WriteBytes(task.GetWriteFile(), task.GetWriteBytes()) + if err != nil { + return nil, err + } + } + + // Create the payload + payload := make([]byte, task.ResultSize) + if _, err = crand.Read(payload); err != nil { + return nil, err + } + + totalMicros := time.Since(startTime).Microseconds() + + // Create the result + r := &LoadResult_Status{ + Id: task.Id, + Payload: payload, + ComputeMicros: computeMicros, + TotalMicros: totalMicros, + FilesRead: filesRead, + BytesRead: bytesRead, + BytesWritten: bytesWritten, + } + + // Record work executed + slog.Info("work executed", + "id", task.Id, + "worktype", worktype, + "computeMicros", computeMicros, + "totalMicros", totalMicros, + "filesRead", filesRead, + "bytesRead", bytesRead, + "bytesWritten", bytesWritten, + ) + + slog.Debug("Returning QuantResult", "result", loggableProto{r}) + + return r, status +} + +// simulateWork +// +// Crash (exit immediately) and fail (with error) have a percentage likelihood, +// min_micros to max_micros will determine how much (randomly) will have 100% +// CPU consumed. +// +// Number of microseconds consumed and error (based on perc_fail) returned. +func simulateWork( + perc_crash float64, + perc_fail float64, + min_micros int64, + max_micros int64) (int64, error) { + + // Calculate the delay + var busyTime int64 + if min_micros > 0 { + busyTime = min_micros + if max_micros > min_micros { + busyTime += int64(rand.Float64() * float64(max_micros-min_micros)) + } + slog.Debug("Working", "min_micros", min_micros, "max_micros", max_micros, "busyTime", busyTime) + if err := busyWork(busyTime); err != nil { + return 0, err + } + } + + // Status of the job + status := rand.Float64() + + // Crash if needed + if status < perc_crash { + slog.Warn("Crashing.") + os.Exit(100) + } + + if status < perc_crash+perc_fail { + return busyTime, fmt.Errorf("job failed") + } + + return busyTime, nil +} + +// busyWork +// +// Consume 100% available CPU until the alloted micros has elapsed. +// +// Error should generally never occur. +func busyWork(micros int64) error { + h := crc32.NewIEEE() + endTime := time.Now().Add(time.Microsecond * time.Duration(micros)) + buf := make([]byte, 32) + + // Loop as long as the current time is NOT after endTime + for !time.Now().After(endTime) { + // Read in random data + _, err := crand.Read(buf) + if err != nil { + return fmt.Errorf("failed reading random bytes: %w", err) + } + + // Write random data to hash function + _, err = h.Write(buf) + if err != nil { + return fmt.Errorf("failed writing to hash: %w", err) + } + } + + return nil +} diff --git a/5-appinfra/modules/htc-infra/loadtest/utils.go b/5-appinfra/modules/htc-infra/loadtest/utils.go new file mode 100644 index 000000000..b6dd3daeb --- /dev/null +++ b/5-appinfra/modules/htc-infra/loadtest/utils.go @@ -0,0 +1,516 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "compress/gzip" + "context" + crand "crypto/rand" + "fmt" + "io" + "io/fs" + "iter" + "log/slog" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "cloud.google.com/go/storage" + "github.com/schollz/progressbar/v3" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +// Run the work, applied to a sequence, with a pool of parallel workers. +// +// The first error will stop the processing of new work, and, once the workers +// are all finished, the first non-nil error is returned. +// +// If there are no errors nil is returned. +func ApplyParallel[V any](seq iter.Seq[V], workers int, work func(v V) error) (err error) { + err = nil + + if workers <= 0 { + workers = runtime.NumCPU() + } + + err_ch := make(chan error, workers) + + working := 0 + + for v := range seq { + + // If at limit, pull an error first. + if working == workers { + last_err := <-err_ch + if last_err != nil { + working-- + err = last_err + break + } + } else { + working++ + } + + // Dispatch the work + go func() { + err_ch <- work(v) + }() + } + + for working > 0 { + last_err := <-err_ch + working-- + if last_err != nil && err == nil { + err = last_err + } + } + + return +} + +// An extension of ApplyParallel, except with a totalWork metric and workFunc needs to return +// the portion of work done as well as error. +// +// A Progress bar is shown if totalWork is > 0. Statistics are logged with elapsed time, total elapsed time (across all workers), +// and overall statistics. +func ApplyParallelWithStats[V any](seq iter.Seq[V], workers int, totalWork int64, workFunc func(v V) (int64, error)) (err error) { + + // Progress bar + var bar *progressbar.ProgressBar + if totalWork > 0 { + bar = progressbar.DefaultBytes(totalWork) + defer func() { + if err := bar.Clear(); err != nil { + slog.Warn("progressbar Clear() error", "error", err) + } + }() + } + + startTime := time.Now() + ioTime := atomic.Int64{} + ops := atomic.Int64{} + work := atomic.Int64{} + + err = ApplyParallel(seq, workers, func(v V) error { + startOpTime := time.Now() + thiswork, err := workFunc(v) + ioTime.Add(time.Since(startOpTime).Nanoseconds()) + work.Add(thiswork) + ops.Add(1) + if totalWork > 0 { + if err := bar.Add64(thiswork); err != nil { + slog.Warn("progress bar Add64() error", "error", err) + } + } + return err + }) + elapsedTime := float64(time.Since(startTime).Nanoseconds()) / 1e9 + totalElapsedTime := float64(ioTime.Load()) / 1e9 + + slog.Info("statistics", "count", ops.Load(), "total", work.Load(), "parallel", workers, "seconds", elapsedTime, "ioseconds", totalElapsedTime, "error", err) + + return err +} + +// WriteLines writes the iterator of strings to the output. It appends newline to the end of each line. +// +// output can be "-" for standard out, and if the output ends with .gz the output will be compressed with gzip. +func WriteLines(output string, it iter.Seq[string]) error { + + var w io.Writer + if output == "-" { + w = os.Stdout + } else { + + if err := MkdirAll(path.Dir(output)); err != nil { + return fmt.Errorf("failed making directory: %v", err) + } + + f, err := CreateWriter(output) + if err != nil { + return fmt.Errorf("error creating: %v", err) + } + + defer func() { + if err := f.Close(); err != nil { + fmt.Printf("Error closing file: %v", err) + } + }() + + w = f + + if strings.HasSuffix(output, ".gz") { + gf := gzip.NewWriter(f) + defer func() { + if err := gf.Close(); err != nil { + fmt.Printf("Error closing file: %v", err) + } + }() + w = gf + } + } + + for line := range it { + _, err := w.Write([]byte(line)) + if err != nil { + return fmt.Errorf("error writing: %v", err) + } + _, err = w.Write([]byte("\n")) + if err != nil { + return fmt.Errorf("error writing: %v", err) + } + } + + return nil +} + +// ReadLines returns an iterator of lines (including the newline) from input. +// +// Input can be "-" for standard in and if the input ends with .gz it is uncompressed. +func ReadLines(input string) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + slog.Info("Opening file", "input", input) + var r io.Reader + if input == "-" { + r = os.Stdin + } else { + file, err := OpenReader(input) + if err != nil { + yield("", fmt.Errorf("error opening %s: %w", input, err)) + return + } + r = file + + defer func() { + if err := file.Close(); err != nil { + fmt.Printf("Error closing file: %v", err) + } + }() + + if strings.HasSuffix(input, ".gz") { + file, err := gzip.NewReader(file) + if err != nil { + yield("", err) + return + } + + defer func() { + if err := file.Close(); err != nil { + fmt.Printf("Error closing file: %v", err) + } + }() + + r = file + } + } + + // Start scanner + scanner := bufio.NewScanner(r) + for scanner.Scan() { + if !yield(scanner.Text(), nil) { + return + } + } + + if err := scanner.Err(); err != nil { + yield("", err) + } + } +} + +func ReadBytesFromDir(dir string) (int64, int64, error) { + slog.Debug("Reading directory", "dir", dir) + + totalBytes := int64(0) + totalFiles := int64(0) + + err := WalkDirFiles(dir, false, func(path string, size int64) error { + slog.Debug("Handling path", "path", path) + cnt, err := ReadBytes(path) + if err != nil { + return err + } + totalBytes += cnt + totalFiles += 1 + return nil + }) + + return totalFiles, totalBytes, err +} + +const bufferSize = 1024 * 1024 + +// Utility to get the storage client (caching) +var storageClient = sync.OnceValues(func() (*storage.Client, error) { + return storage.NewClient(context.Background(), option.WithUserAgent( + "cloud-solutions/fsi-rdp-loadtest-v1.0.0")) +}) + +var gsPattern = regexp.MustCompile(`^gs://([^/]+)/(.*)$`) + +// Mkdir skips for gs:// outputs +func MkdirAll(path string) error { + gs_match := gsPattern.FindStringSubmatch(path) + if gs_match != nil { + return nil + } + + return os.MkdirAll(path, 0750) +} + +// Join paths together, but handles gs:// prefixes +func Join(path1 string, path2 string) string { + if strings.HasPrefix(path1, "gs://") { + if !strings.HasSuffix(path1, "/") { + return path1 + "/" + path2 + } else { + return path1 + path2 + } + } + return filepath.Join(path1, path2) +} + +// WalkDirFiles handles local files or gs:// buckets and objects +func WalkDirFiles(dir string, includeSize bool, handle func(file string, size int64) error) error { + gs_match := gsPattern.FindStringSubmatch(dir) + if gs_match != nil { + client, err := storageClient() + if err != nil { + return fmt.Errorf("opening creating GCS client: %w", err) + } + if !strings.HasSuffix(gs_match[2], "/") { + gs_match[2] = gs_match[2] + "/" + } + slog.Debug("listing GCS objects", "bucket", gs_match[1], "prefix", gs_match[2]) + it := client.Bucket(gs_match[1]).Objects(context.Background(), &storage.Query{ + Prefix: gs_match[2], + Versions: false, + Projection: storage.ProjectionNoACL}) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("listing objects in path %s: %w", dir, err) + } + if err := handle(fmt.Sprintf("gs://%s/%s", attrs.Bucket, attrs.Name), attrs.Size); err != nil { + return err + } + } + return nil + } + + return filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + slog.Debug("Handling path", "path", path, "d", d, "err", err) + if err != nil { + return err + } + if d.IsDir() { + return nil + } + size := int64(0) + if includeSize { + info, err := d.Info() + if err != nil { + return err + } + size = info.Size() + } + if err := handle(path, size); err != nil { + return err + } + return nil + }) +} + +// Open a reader from local POSIX or from GCS +func OpenReader(file string) (io.ReadCloser, error) { + + // ReadCloser + var r io.ReadCloser + + // if starts with gs://, then open an object instead. + gs_match := gsPattern.FindStringSubmatch(file) + if gs_match != nil { + client, err := storageClient() + if err != nil { + return nil, fmt.Errorf("opening creating GCS client: %w", err) + } + f, err := client.Bucket(gs_match[1]).Object(gs_match[2]).NewReader(context.Background()) + if err != nil { + return nil, fmt.Errorf("reading GCS object: %w", err) + } + r = f + } else { + // Open normal file + f, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("opening file %s: %w", file, err) + } + r = f + } + + return r, nil +} + +// Open a writer to local POSIX or to GCS +func CreateWriter(file string) (io.WriteCloser, error) { + // if starts with gs://, then open an object instead. + gs_match := gsPattern.FindStringSubmatch(file) + if gs_match != nil { + client, err := storageClient() + if err != nil { + return nil, fmt.Errorf("opening creating GCS client: %w", err) + } + slog.Debug("Opening GCS for writing", "bucket", gs_match[1], "object", gs_match[2]) + o := client.Bucket(gs_match[1]).Object(gs_match[2]).NewWriter(context.Background()) + return o, nil + } else { + err := MkdirAll(filepath.Dir(file)) + if err != nil { + return nil, fmt.Errorf("error creating directory %s: %w", filepath.Dir(file), err) + } + + slog.Debug("Opening file for writing", "file", file) + o, err := os.Create(file) + if err != nil { + return nil, fmt.Errorf("error opening file %s for writing: %w", file, err) + } + return o, nil + } +} + +// Read file and return the bytes read and error +func ReadBytes(file string) (int64, error) { + slog.Debug("Reading", "file", file) + f, err := OpenReader(file) + if err != nil { + return 0, err + } + buf := make([]byte, bufferSize) + var bytesRead int64 = 0 + for { + r, err := f.Read(buf) + bytesRead += int64(r) + if err != nil { + if err == io.EOF { + break + } + return bytesRead, err + } + } + if err := f.Close(); err != nil { + return bytesRead, err + } + + return bytesRead, nil +} + +// Write random data to file, of size bytes, and return bytes written and error +func WriteBytes(file string, size int64) (int64, error) { + slog.Debug("Writing", "file", file, "size", size) + + f, err := CreateWriter(file) + if err != nil { + return 0, err + } + + sizeLeft := size + + buf := make([]byte, bufferSize) + for sizeLeft > 0 { + + // Adjust size of buffer if nearly complete + if len(buf) > int(sizeLeft) { + buf = buf[:sizeLeft] + } + + // Read in random data + _, err := crand.Read(buf) + if err != nil { + return size - sizeLeft, fmt.Errorf("failed generating random bytes") + } + + // Write random data to output file + s, err := f.Write(buf) + if err != nil { + return size - sizeLeft, fmt.Errorf("failed writing") + } + + sizeLeft -= int64(s) + } + + // Close file + if err := f.Close(); err != nil { + return size, err + } + + return size, nil +} + +/* + * Utility for limiting the number of concurrent workers + */ + +type workCounter struct { + maxWorkers int // Maximum number of workers + currentWorkers int // Current number of workers + workerLock *sync.Cond // Mutex condition for releasing when free +} + +// NewWorkerCounter creates a work counter limiting to maxWorkers +func NewWorkerCounter(maxWorkers int) *workCounter { + return &workCounter{ + maxWorkers: maxWorkers, + currentWorkers: 0, + workerLock: sync.NewCond(&sync.Mutex{}), + } +} + +// Acquire a free worker slot. +// +// Will return immediately if available, +// otherwise will block until one is free. +// +// Workers should Acquire and Release within the function: +// w.Acquire() +// defer w.Release() +func (w *workCounter) Acquire() { + w.workerLock.L.Lock() + for w.currentWorkers >= w.maxWorkers { + w.workerLock.Wait() + } + w.currentWorkers++ + w.workerLock.L.Unlock() +} + +// Release worker slot. +// +// Will let another worker proceed if one is blocked in Acquire(). +func (w *workCounter) Release() { + w.workerLock.L.Lock() + w.currentWorkers-- + w.workerLock.Signal() + w.workerLock.L.Unlock() +} diff --git a/5-appinfra/modules/htc-infra/main.tf b/5-appinfra/modules/htc-infra/main.tf new file mode 100644 index 000000000..faac25ddf --- /dev/null +++ b/5-appinfra/modules/htc-infra/main.tf @@ -0,0 +1,358 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + namespace = "${var.team}-${var.env}" + storage_locations_map = length(var.storage_locations) > 0 ? var.storage_locations : { + for region in var.regions : region => region + } + workload_args = ["serve", "--logJSON"] + workload_grpc_endpoint = "http://localhost:2002/main.LoadTestService/RunLibrary" + workload_init_args = [ + ["writedata", "--logJSON", "--showProgress=false", "--size", "1048576", "--parallel", "20", "--count", "5", "/data/read_dir/small-low"], + ["gentasks", "--logJSON", "--count=1000", "--initMinMicros=1000000", "--minMicros=1000000", "--initReadDir=/data/read_dir/small", "/data/tasks/1k_1s_1s_read_small.jsonl.gz"], + ] + test_configs = [ + { name = "hpa_1k_1s_s_read_small", testfile = "/data/tasks/1k_1s_1s_read_small.jsonl.gz", parallel = 0, description = "Autoscaling workers, 1 second compute and small folder read on initialization, 1,000 tasks for 1 second task compute." }, + ] + test_configs_dict = { + for config in local.test_configs : + config.name => config + } + + ui_config_file = yamlencode({ + "project_id" : var.infra_project, + "region" : var.region, + "pubsub_summary_table" : "${google_bigquery_table.messages_summary.project}.${google_bigquery_table.messages_summary.dataset_id}.${google_bigquery_table.messages_summary.table_id}", + "urls" : { + "dashboard" = module.gke.monitoring_dashboard_url + "cluster" = module.gke.cluster_urls + }, + "tasks" : concat( + length(module.gke) == 0 ? [] : [ + for config in local.test_configs : { + "name" = "GKE ${config.name}", + # "script" = module.gke.first_test_script[config.name], + "script" = module.gke.test_scripts_list[0], + "parallel" = config.parallel, + "description" = config.description, + } + ], + ), + }) + parallelstore_instances = var.storage_type == "PARALLELSTORE" ? { + for region, instance in module.parallelstore : region => { + name = instance.name_short + instance_id = instance.instance_id + access_points = instance.access_points + location = instance.location + region = instance.region + id = instance.id + capacity_gib = instance.capacity_gib + daos_version = instance.daos_version + kubernetes_usage = { + persistent_volume_claim_template = "templates/parallelstore-pvc.yaml" + csi_driver_enabled = var.enable_csi_parallelstore + } + } + } : {} +} + +#----------------------------------------------------- +# Container Image Building +#----------------------------------------------------- + +module "agent" { + source = "./modules/builder" + + project_id = var.admin_project + region = var.region + repository_region = var.region + repository_id = var.service_name + + containers = { + agent = { + source = "${path.module}/agent/src" + }, + loadtest = { + source = "${path.module}/loadtest" + }, + american_option = { + source = "${path.module}/american-option" + } + } +} + +#----------------------------------------------------- +# GKE Deployment +#----------------------------------------------------- + +module "gke" { + source = "./agent/modules/gke" + gke_cluster_names = var.gke_cluster_names + project_id = var.infra_project + cluster_project_id = var.cluster_project_id + regions = ["us-central1"] + agent_image = module.agent.status["agent"].image_url + namespace = local.namespace + env = var.env + # GCS specific options + hsn_bucket = var.hsn_bucket + + # Pub/Sub configuration + pubsub_exactly_once = var.pubsub_exactly_once + + # Workload options + # TODO: Other configuration for the workload - needs to be standardized + workload_image = module.agent.status["loadtest"].image_url + workload_args = local.workload_args + workload_grpc_endpoint = local.workload_grpc_endpoint + workload_init_args = local.workload_init_args + test_configs = local.test_configs_dict + + parallelstore_enabled = var.storage_type == "PARALLELSTORE" + parallelstore_instances = local.parallelstore_instances + vpc_name = var.network_name + + depends_on = [ + module.agent + ] +} + +#----------------------------------------------------- +# Logging and Analytics Configuration +#----------------------------------------------------- + +# Always have the same filter. Leave this at the top level. This will +# capture application *and* agent stuff. +# To be put into an *agent* module... +resource "google_logging_linked_dataset" "logging_linked_dataset" { + link_id = "applogs" + bucket = google_logging_project_bucket_config.analytics-enabled-bucket.id + description = "Linked dataset test" +} + +resource "google_logging_project_bucket_config" "analytics-enabled-bucket" { + project = var.infra_project + location = var.region + enable_analytics = true + bucket_id = "applogs" + lifecycle { + ignore_changes = [project] + } +} + +resource "google_logging_project_sink" "my-sink" { + project = var.infra_project + name = "my-pubsub-instance-sink" + + # Can export to pubsub, cloud storage, bigquery, log bucket, or another project + destination = "logging.googleapis.com/${google_logging_project_bucket_config.analytics-enabled-bucket.id}" + + # This depends on the use of GKE and Cloud Run for the filter + filter = "logName=\"projects/${var.infra_project}/logs/stdout\" OR logName=\"projects/${var.infra_project}/logs/stderr\" OR logName=\"projects/${var.infra_project}/logs/run.googleapis.com%2Fstdout\" OR logName=\"projects/${var.infra_project}/logs/run.googleapis.com%2Fstdout\"" + + description = "Local application logs" +} + +resource "google_bigquery_dataset" "main" { + project = var.infra_project + dataset_id = "workload" + location = var.region + delete_contents_on_destroy = true + + depends_on = [module.enabled_google_apis] +} + +# +# Logging BigQuery views +# +resource "google_bigquery_table" "log_stats" { + project = var.infra_project + dataset_id = google_bigquery_dataset.main.dataset_id + table_id = "log_stats" + deletion_protection = false + + view { + query = templatefile( + "${path.module}/sql/log_stats.sql", { + project_id = var.infra_project + dataset_id = regex("^bigquery.googleapis.com/projects/[^/]+/datasets/(.*)$", google_logging_linked_dataset.logging_linked_dataset.bigquery_dataset[0].dataset_id)[0] + }) + use_legacy_sql = false + } +} + +# Collect agent statistics +resource "google_bigquery_table" "agent_stats" { + project = var.infra_project + dataset_id = google_bigquery_dataset.main.dataset_id + table_id = "agent_stats" + deletion_protection = false + + view { + query = templatefile( + "${path.module}/sql/agent_stats.sql", { + project_id = var.infra_project + dataset_id = google_bigquery_dataset.main.dataset_id + table_id = google_bigquery_table.log_stats.table_id + }) + use_legacy_sql = false + } +} + +# Summarise agent statistics by instance +resource "google_bigquery_table" "agent_summary_by_instance" { + project = var.infra_project + dataset_id = google_bigquery_dataset.main.dataset_id + table_id = "agent_summary_by_instance" + deletion_protection = false + + view { + query = templatefile( + "${path.module}/sql/agent_summary_by_instance.sql", { + project_id = var.infra_project + dataset_id = google_bigquery_dataset.main.dataset_id + table_id = google_bigquery_table.agent_stats.table_id + }) + use_legacy_sql = false + } +} + + +#----------------------------------------------------- +# PubSub and BigQuery Integration +#----------------------------------------------------- +# All message are assumed to be JSON and captured in a JSON data element +module "bigquery_capture" { + source = "./modules/pubsub-subscriptions" + project_id = var.infra_project + # region = var.region + bigquery_dataset = google_bigquery_dataset.main.dataset_id + bigquery_table = "pubsub_messages" + subscriber_service_account = google_service_account.bq_write_service_account.email + topics = concat( + length(module.gke) > 0 ? module.gke.topics : [] + ) +} + +resource "google_pubsub_topic_iam_member" "topic_subscriber" { + for_each = toset(concat( + length(module.gke) > 0 ? module.gke.topics : [] + )) + + project = var.infra_project + topic = each.value + role = "roles/pubsub.subscriber" + member = "serviceAccount:${google_service_account.bq_write_service_account.email}" +} + +resource "google_service_account" "bq_write_service_account" { + project = var.infra_project + account_id = "pubsub-bigquery-writer" + display_name = "BQ Write Service Account" +} + +#----------------------------------------------------- +# Quota Requests +#----------------------------------------------------- + +module "quota" { + count = var.additional_quota_enabled ? 1 : 0 + source = "./modules/quota" + + project_id = var.infra_project + quota_contact_email = var.quota_contact_email + + quota_preferences = [ + { + service = "compute.googleapis.com" + quota_id = "PREEMPTIBLE-CPUS-per-project-region" + preferred_value = 10000 + region = var.region + }, + { + service = "compute.googleapis.com" + quota_id = "DISKS-TOTAL-GB-per-project-region" + preferred_value = 65000 + region = var.region + }, + { + service = "monitoring.googleapis.com" + quota_id = "IngestionRequestsPerMinutePerProject" + preferred_value = 100000 + } + ] +} + +#----------------------------------------------------- +# BigQuery Views for Visualization +#----------------------------------------------------- + +# Pub/Sub messages joined by request/response +resource "google_bigquery_table" "messages_joined" { + project = var.infra_project + dataset_id = google_bigquery_dataset.main.dataset_id + table_id = "pubsub_messages_joined" + deletion_protection = false + + view { + query = templatefile( + "${path.module}/sql/pubsub_messages_joined.sql", { + project_id = var.infra_project + dataset_id = google_bigquery_dataset.main.dataset_id + table_id = "pubsub_messages" + }) + use_legacy_sql = false + } + + depends_on = [ + module.bigquery_capture + ] +} + +# Pub/Sub summary by job +resource "google_bigquery_table" "messages_summary" { + project = var.infra_project + dataset_id = google_bigquery_dataset.main.dataset_id + table_id = "pubsub_messages_summary" + deletion_protection = false + + view { + query = templatefile( + "${path.module}/sql/pubsub_messages_summary.sql", { + project_id = google_bigquery_table.messages_joined.project + dataset_id = google_bigquery_table.messages_joined.dataset_id + joined_table_id = google_bigquery_table.messages_joined.table_id + }) + use_legacy_sql = false + } + + depends_on = [ + google_bigquery_table.messages_joined + ] +} + +# Create Parallel Store +module "parallelstore" { + for_each = var.storage_type == "PARALLELSTORE" ? local.storage_locations_map : {} + source = "./modules/parallelstore" + project_id = var.infra_project + location = var.region + network = var.network_self_link + capacity_gib = var.storage_capacity_gib + deployment_type = var.parallelstore_deployment_type + + depends_on = [module.enabled_google_apis] +} diff --git a/5-appinfra/modules/htc-infra/modules/artifact-registry/README.md b/5-appinfra/modules/htc-infra/modules/artifact-registry/README.md new file mode 100644 index 000000000..2d1451b8e --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/artifact-registry/README.md @@ -0,0 +1,83 @@ +# Google Artifact Registry Module + +This module creates a Google Artifact Registry repository to store container images for risk and research workloads. It configures the repository with appropriate cleanup policies to manage image versions. + +## Usage + +```hcl +module "artifact_registry" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/artifact-registry" + + project_id = "your-project-id" + regions = ["us-central1", "us-east4"] + name = "research-images" + cleanup_keep_count = 10 +} +``` + +## Features + +- Creates a Docker container registry in Artifact Registry +- Automatically determines the best multi-region location based on provided regions +- Configures cleanup policies to maintain a specific number of recent image versions +- Enables vulnerability scanning for container images + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|----------| +| project_id | The GCP project where the resources will be created | `string` | n/a | yes | +| regions | List of regions where resources will be deployed - used to determine the multi-region location | `list(string)` | `["us-central1"]` | no | +| name | Name of the Artifact Registry | `string` | `"research-images"` | no | +| cleanup_keep_count | Number of most recent container image versions to keep in Artifact Registry cleanup policy | `number` | `10` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| artifact_registry | The Artifact Registry repository object | +| location | The location of the Artifact Registry repository | +| repository_url | The URL of the Artifact Registry repository | + +## Notes + +- The module automatically determines whether to use a regional or multi-regional location for the repository based on the regions provided. +- Cleanup policies are configured to keep the specified number of most recent container image versions. +- Vulnerability scanning is enabled by default to enhance container security. + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cleanup\_keep\_count | Number of most recent container image versions to keep in Artifact Registry cleanup policy | `number` | `10` | no | +| name | Name of the Artifact Registry | `string` | `"research-images"` | no | +| project\_id | The GCP project where the resources will be created | `string` | n/a | yes | +| regions | List of regions where GKE clusters will be deployed - used to determine the multi-region location | `list(string)` | 
[
  "us-central1"
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| artifact\_registry | Complete Artifact Registry repository resource including all attributes and properties | +| artifact\_registry\_id | Fully qualified ID of the Artifact Registry repository in format projects/{project}/locations/{location}/repositories/{repository} | +| artifact\_registry\_region | Region where the Artifact Registry repository was deployed based on dominant region analysis | +| artifact\_registry\_url | Full URL of the Artifact Registry repository for use in Docker image references (format: {region}-docker.pkg.dev/{project}/{repository}) | + + diff --git a/5-appinfra/modules/htc-infra/modules/artifact-registry/main.tf b/5-appinfra/modules/htc-infra/modules/artifact-registry/main.tf new file mode 100644 index 000000000..af8a4dfc6 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/artifact-registry/main.tf @@ -0,0 +1,46 @@ + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_project" "environment" { + project_id = var.project_id +} + +module "region_analysis" { + source = "../region-analysis" + regions = var.regions +} + +# Artifact Registry +resource "google_artifact_registry_repository" "research-images" { + project = var.project_id + location = module.region_analysis.dominant_region + repository_id = var.name + description = "Multi-region artifact registry in ${module.region_analysis.dominant_region}" + format = "DOCKER" + # Keep only the most recent image versions based on cleanup_keep_count + cleanup_policies { + id = "keep-minimum-versions" + action = "KEEP" + most_recent_versions { + keep_count = var.cleanup_keep_count + } + } + lifecycle { + precondition { + condition = module.region_analysis.region_counts[module.region_analysis.dominant_region] > 0 + error_message = "Could not determine appropriate multi-region location from provided regions: ${jsonencode(var.regions)}" + } + } +} diff --git a/5-appinfra/modules/htc-infra/modules/artifact-registry/outputs.tf b/5-appinfra/modules/htc-infra/modules/artifact-registry/outputs.tf new file mode 100644 index 000000000..58bfceb9b --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/artifact-registry/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "artifact_registry_url" { + description = "Full URL of the Artifact Registry repository for use in Docker image references (format: {region}-docker.pkg.dev/{project}/{repository})" + value = "${module.region_analysis.dominant_region}-docker.pkg.dev/${data.google_project.environment.project_id}/${google_artifact_registry_repository.research-images.name}" +} + +output "artifact_registry" { + description = "Complete Artifact Registry repository resource including all attributes and properties" + value = google_artifact_registry_repository.research-images +} + +output "artifact_registry_id" { + description = "Fully qualified ID of the Artifact Registry repository in format projects/{project}/locations/{location}/repositories/{repository}" + value = google_artifact_registry_repository.research-images.id +} + +output "artifact_registry_region" { + description = "Region where the Artifact Registry repository was deployed based on dominant region analysis" + value = module.region_analysis.dominant_region +} diff --git a/5-appinfra/modules/htc-infra/modules/artifact-registry/variables.tf b/5-appinfra/modules/htc-infra/modules/artifact-registry/variables.tf new file mode 100644 index 000000000..80338c749 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/artifact-registry/variables.tf @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The GCP project where the resources will be created" + type = string + + validation { + condition = var.project_id != "YOUR_PROJECT_ID" + error_message = "'project_id' was not set, please set the value in the fsi-resaerch-1.tfvars file" + } +} + +# Region for resource deployment (default: us-central1) +variable "regions" { + description = "List of regions where GKE clusters will be deployed - used to determine the multi-region location" + type = list(string) + default = ["us-central1"] +} + +variable "name" { + description = "Name of the Artifact Registry" + type = string + default = "research-images" +} + +variable "cleanup_keep_count" { + description = "Number of most recent container image versions to keep in Artifact Registry cleanup policy" + type = number + default = 10 +} diff --git a/5-appinfra/modules/htc-infra/modules/artifact-registry/versions.tf b/5-appinfra/modules/htc-infra/modules/artifact-registry/versions.tf new file mode 100644 index 000000000..1eac1780d --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/artifact-registry/versions.tf @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.4" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/builder/README.md b/5-appinfra/modules/htc-infra/modules/builder/README.md new file mode 100644 index 000000000..4668cc771 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/builder/README.md @@ -0,0 +1,106 @@ +# Google Cloud Build Module + +This module configures and executes Cloud Build jobs to build and push container images to Artifact Registry for risk and research workloads. + +## Usage + +```hcl +module "builder" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/builder" + + project_id = "your-project-id" + region = "us-central1" + repository_region = "us-central1" + repository_id = "research-images" + + containers = { + app1 = { + source = "${path.module}/src/app1" + }, + app2 = { + source = "${path.module}/src/app2" + config_yaml = file("${path.module}/config/app2-config.yaml") + } + } + + service_account_name = "cloudbuild-sa" +} +``` + +## Features + +- Builds container images from source directories +- Pushes built images to Artifact Registry +- Creates and configures service accounts with appropriate permissions +- Supports passing configuration YAML to builds +- Provides status information about builds and resulting images + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|----------| +| project_id | The GCP project ID where resources will be created | `string` | n/a | yes | +| region | The region of the build | `string` | n/a | yes | +| repository_region | Artifact Repository region | `string` | n/a | yes | +| repository_id | Artifact repository ID | `string` | n/a | yes | +| containers | Map of image name to configuration (source) | `map(object)` | n/a | yes | +| service_account_name | Service account name | `string` | `"cloudbuild-actor"` | no | + +### Containers Object Structure + +```hcl +map(object({ + source = string # Path to the source directory containing Dockerfile + config_yaml = string # Optional: Configuration YAML to pass to the build +})) +``` + +## Outputs + +| Name | Description | +|------|-------------| +| status | Map of container build status information | +| service_account | The service account created for Cloud Build | + +## Notes + +- Each source directory should contain a Dockerfile or cloudbuild.yaml +- The module automatically creates a service account with appropriate permissions +- Images are tagged with the short commit hash by default +- Build logs are available in Cloud Build history +- Images are pushed to the specified Artifact Registry repository + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| containers | Map of image name to configuration (source) | 
map(object({
    source      = string
    config_yaml = optional(string, "")
  }))
| n/a | yes | +| project\_id | The GCP project ID where resources will be created. | `string` | n/a | yes | +| region | The region of the build | `string` | n/a | yes | +| repository\_id | Artifact repository ID | `string` | n/a | yes | +| repository\_region | Artifacte Repository region | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| status | Container build status information including image URLs, tags, and build metadata for all containers built by Cloud Build | + + diff --git a/5-appinfra/modules/htc-infra/modules/builder/main.tf b/5-appinfra/modules/htc-infra/modules/builder/main.tf new file mode 100644 index 000000000..11f84dc26 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/builder/main.tf @@ -0,0 +1,117 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + repository_prefix = "${var.repository_region}-docker.pkg.dev/${var.project_id}/${var.repository_id}" + container_hash = { + for container, config in var.containers : + container => sha512(join("", concat([ + for f in fileset(config.source, "**/*") : fileexists("${config.source}/${f}") ? filesha512("${config.source}/${f}") : sha512("") + ], [ + config.config_yaml == "" ? "" : sha512(config.config_yaml) + ]))) + } + container_tag = { + for container, config in var.containers : + container => "ver-${substr(local.container_hash[container], 0, 10)}" + } + container_image = { + for container, config in var.containers : + container => "${local.repository_prefix}/${container}:${local.container_tag[container]}" + } + container_status = { + for container, config in var.containers : + container => merge(config, { + "tag" = local.container_tag[container], + "image" = local.container_image[container], + "hash" = local.container_hash[container], + "config_yaml_file" = config.config_yaml == "" ? "" : "config.yaml", + }) + } +} + +# +# Create Cloud Build staging bucket and grant permissions +# + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +resource "google_storage_bucket" "cloudbuild" { + project = var.project_id + location = var.region + name = "${var.project_id}-${var.region}-cloudbuild-${random_string.suffix.id}" + uniform_bucket_level_access = true + + force_destroy = true +} + + +# +# Launch CloudBuild +# + +resource "null_resource" "run_cloud_build" { + for_each = var.containers + + triggers = { + source_contents_hash = local.container_hash[each.key] + } + + provisioner "local-exec" { + when = create + command = <<-EOT + + # Exit on any error + set -e + + # Write to config.yaml + if [ "${local.container_status[each.key].config_yaml_file}" != "" ]; then + cat > "${each.value.source}/${local.container_status[each.key].config_yaml_file}" < merge(status, { + "image_url" = data.google_artifact_registry_docker_image.final_image[container].self_link + }) + } + + depends_on = [ + null_resource.run_cloud_build + ] +} diff --git a/5-appinfra/modules/htc-infra/modules/builder/variables.tf b/5-appinfra/modules/htc-infra/modules/builder/variables.tf new file mode 100644 index 000000000..1f077ee23 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/builder/variables.tf @@ -0,0 +1,53 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Project ID where resources will be deployed +variable "project_id" { + type = string + description = "The GCP project ID where resources will be created." +} + +# Region where the build and artifact repository is +variable "region" { + type = string + description = "The region of the build" +} + +variable "repository_region" { + type = string + description = "Artifacte Repository region" + +} + +# Repository ID +variable "repository_id" { + type = string + description = "Artifact repository ID" +} + +# Containers to build +variable "containers" { + type = map(object({ + source = string + config_yaml = optional(string, "") + })) + description = "Map of image name to configuration (source)" +} + +# Service account name to create +# variable "service_account_name" { +# type = string +# description = "Service account name" +# default = "cloudbuild-actor" +# } diff --git a/5-appinfra/modules/htc-infra/modules/builder/versions.tf b/5-appinfra/modules/htc-infra/modules/builder/versions.tf new file mode 100644 index 000000000..1eac1780d --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/builder/versions.tf @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.4" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/kubectl/README.md b/5-appinfra/modules/htc-infra/modules/kubectl/README.md new file mode 100644 index 000000000..0b5cbbc7d --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/kubectl/README.md @@ -0,0 +1,60 @@ +# Kubectl Module + +This module provides version constraints for Terraform providers used in Kubernetes deployments via kubectl, ensuring consistent provider versions across the blueprint. + +## Usage + +```hcl +module "kubectl_config" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/kubectl" +} +``` + +## Features + +- Defines consistent version constraints for Google Cloud providers +- Ensures compatibility with Kubernetes resources +- Establishes a standard provider configuration for modules that interact with kubectl + +## Providers + +This module sets version constraints for the following providers: + +| Provider | Version | +|----------|---------| +| google | ~> 6.29.0 | +| google-beta | ~> 6.29.0 | + +## Notes + +- This module primarily serves as a provider configuration reference +- Include this module in Terraform configurations that need to interact with Kubernetes via kubectl +- The module ensures consistent provider versions to avoid compatibility issues +- The provider_meta configuration ensures proper attribution of resources created by this module + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +No inputs. + +## Outputs + +No outputs. + + diff --git a/5-appinfra/modules/htc-infra/modules/kubectl/versions.tf b/5-appinfra/modules/htc-infra/modules/kubectl/versions.tf new file mode 100644 index 000000000..d21257e22 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/kubectl/versions.tf @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/lustre/README.md b/5-appinfra/modules/htc-infra/modules/lustre/README.md new file mode 100644 index 000000000..4d8916911 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/lustre/README.md @@ -0,0 +1,100 @@ +# Google Cloud Lustre Module + +This module creates a Lustre filesystem on Google Cloud, a high-performance parallel file system designed for HPC, genomics, and other data-intensive workloads. + +## Usage + +```hcl +module "lustre" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/lustre" + + project_id = "your-project-id" + location = "us-central1-a" + network = google_compute_network.vpc.id + filesystem = "lustre-fs" + capacity_gib = 18000 + gke_support_enabled = true +} +``` + +## Features + +- Create high-performance Lustre filesystems for data-intensive workloads +- GKE integration for using Lustre with Kubernetes workloads +- Configurable capacity in multiples of 9000 GiB +- Automatic zone selection if only a region is provided +- VPC network connectivity for secure access + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|----------| +| project_id | The GCP project where the resources will be created | `string` | n/a | yes | +| location | The location (zone) where the Lustre instance will be created | `string` | `null` | yes | +| instance_id | The ID of the Lustre instance | `string` | `null` | no | +| filesystem | The name of the Lustre filesystem | `string` | `"lustre-fs"` | no | +| network | The VPC network to which the Lustre instance should be connected | `string` | `"default"` | no | +| capacity_gib | Capacity in GiB for Lustre instance. Must be a multiple of 9000. | `number` | `18000` | no | +| gke_support_enabled | Enable GKE support for Lustre instance | `bool` | `true` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| instance | The created Lustre instance | +| instance_id | The ID of the Lustre instance | +| instance_name | The name of the Lustre instance | +| location | The location where the Lustre instance is created | +| filesystem | The name of the Lustre filesystem | + +## Notes + +- Lustre capacity must be a multiple of 9000 GiB between 18000 GiB and 936000 GiB +- GKE integration allows using Lustre as persistent volumes in Kubernetes +- Requires a properly configured VPC network with Private Service Access +- For best performance, ensure the network MTU is set to 8896 +- The module handles automatic selection of a zone if only a region is provided + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| capacity\_gib | Capacity in GiB for Lustre instance. Must be a multiple of 9000. | `number` | `18000` | no | +| filesystem | The name of the Lustre filesystem | `string` | `"lustre-fs"` | no | +| gke\_support\_enabled | Enable GKE support for Lustre instance | `bool` | `true` | no | +| instance\_id | The ID of the Lustre instance. If null, will be set to 'lustre-{location}'. | `string` | `null` | no | +| location | The location (zone) where the Lustre instance will be created, in the format 'region-zone' e.g., 'us-central1-a' | `string` | `"null"` | no | +| network | The VPC network to which the Lustre instance should be connected | `string` | `"default"` | no | +| project\_id | The GCP project where the resources will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| capacity\_gib | Provisioned storage capacity of the Lustre instance in GiB | +| filesystem | Name of the Lustre filesystem | +| id | Fully qualified identifier for the Lustre resource | +| instance\_id | ID of the Lustre instance | +| location | Zone location where the Lustre instance is deployed | +| mount\_point | List of IPv4 addresses or DNS names for the Lustre mount points | +| name | Fully qualified name of the Lustre instance | +| region | Region extracted from the location | + + diff --git a/5-appinfra/modules/htc-infra/modules/lustre/main.tf b/5-appinfra/modules/htc-infra/modules/lustre/main.tf new file mode 100644 index 000000000..65173fab9 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/lustre/main.tf @@ -0,0 +1,54 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + is_zone = can(regex("-[a-z]$", var.location)) + region = local.is_zone ? regex("^(.*)-[a-z]$", var.location)[0] : var.location +} + +# data "google_project" "environment" { +# project_id = var.project_id +# } + +# Get available zones for the region +data "google_compute_zones" "available" { + project = var.project_id + region = local.region +} + +# Random zone selection +resource "random_shuffle" "zone" { + input = data.google_compute_zones.available.names + result_count = 1 +} + +# Create Lustre instance +resource "google_lustre_instance" "lustre" { + provider = google-beta + project = var.project_id + instance_id = var.instance_id != null ? var.instance_id : "lustre-${var.location}" + location = var.location + filesystem = var.filesystem + capacity_gib = var.capacity_gib + network = var.network + gke_support_enabled = var.gke_support_enabled + + per_unit_storage_throughput = 1000 + + timeouts { + create = "120m" + update = "120m" + delete = "120m" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/lustre/outputs.tf b/5-appinfra/modules/htc-infra/modules/lustre/outputs.tf new file mode 100644 index 000000000..eb2a24ffe --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/lustre/outputs.tf @@ -0,0 +1,53 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "id" { + description = "Fully qualified identifier for the Lustre resource" + value = google_lustre_instance.lustre.id +} + +output "name" { + description = "Fully qualified name of the Lustre instance" + value = google_lustre_instance.lustre.name +} + +output "instance_id" { + description = "ID of the Lustre instance" + value = google_lustre_instance.lustre.instance_id +} + +output "mount_point" { + description = "List of IPv4 addresses or DNS names for the Lustre mount points" + value = google_lustre_instance.lustre.mount_point +} + +output "location" { + description = "Zone location where the Lustre instance is deployed" + value = google_lustre_instance.lustre.location +} + +output "region" { + description = "Region extracted from the location" + value = split("-", google_lustre_instance.lustre.location)[0] +} + +output "capacity_gib" { + description = "Provisioned storage capacity of the Lustre instance in GiB" + value = google_lustre_instance.lustre.capacity_gib +} + +output "filesystem" { + description = "Name of the Lustre filesystem" + value = google_lustre_instance.lustre.filesystem +} diff --git a/5-appinfra/modules/htc-infra/modules/lustre/variables.tf b/5-appinfra/modules/htc-infra/modules/lustre/variables.tf new file mode 100644 index 000000000..20924e9d2 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/lustre/variables.tf @@ -0,0 +1,64 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The GCP project where the resources will be created" + type = string + + validation { + condition = var.project_id != "YOUR_PROJECT_ID" + error_message = "'project_id' was not set, please set the value in the terraform.tfvars file" + } +} + +variable "location" { + description = "The location (zone) where the Lustre instance will be created, in the format 'region-zone' e.g., 'us-central1-a'" + type = string + default = "null" +} + +variable "instance_id" { + description = "The ID of the Lustre instance. If null, will be set to 'lustre-{location}'." + type = string + default = null +} + +variable "filesystem" { + description = "The name of the Lustre filesystem" + type = string + default = "lustre-fs" +} + +variable "network" { + description = "The VPC network to which the Lustre instance should be connected" + type = string + default = "default" +} + +variable "capacity_gib" { + description = "Capacity in GiB for Lustre instance. Must be a multiple of 9000." + type = number + default = 18000 + + validation { + condition = var.capacity_gib >= 18000 && var.capacity_gib <= 936000 && var.capacity_gib % 9000 == 0 + error_message = "Capacity must be a multiple of 9000 GiB between 18000 GiB and 936000 GiB" + } +} + +variable "gke_support_enabled" { + description = "Enable GKE support for Lustre instance" + type = bool + default = true +} diff --git a/5-appinfra/modules/htc-infra/modules/lustre/versions.tf b/5-appinfra/modules/htc-infra/modules/lustre/versions.tf new file mode 100644 index 000000000..1eac1780d --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/lustre/versions.tf @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.4" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/parallelstore/README.md b/5-appinfra/modules/htc-infra/modules/parallelstore/README.md new file mode 100644 index 000000000..2e596ca18 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/parallelstore/README.md @@ -0,0 +1,98 @@ +# Google Cloud Parallelstore Module + +This module creates a Google Cloud Parallelstore instance, a high-performance filesystem designed for HPC and ML/AI workloads requiring fast data access across multiple compute nodes. + +## Usage + +```hcl +module "parallelstore" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/parallelstore" + + project_id = "your-project-id" + location = "us-central1-a" + network = google_compute_network.vpc.id + deployment_type = "SCRATCH" + capacity_gib = 12000 +} +``` + +## Features + +- Create Parallelstore instances in either SCRATCH or PERSISTENT deployment types +- Automatically select a zone if only a region is provided +- Customizable capacity based on workload requirements +- Default capacity values optimized for each deployment type +- VPC network connectivity for secure access + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|----------| +| project_id | The GCP project where the resources will be created | `string` | n/a | yes | +| location | The location (zone) where the Parallelstore instance will be created | `string` | `null` | yes | +| instance_id | The ID of the Parallelstore instance | `string` | `null` | no | +| network | The VPC network to which the Parallelstore instance should be connected | `string` | `"default"` | no | +| deployment_type | Parallelstore Instance deployment type (SCRATCH or PERSISTENT) | `string` | `"SCRATCH"` | no | +| capacity_gib | Custom capacity in GiB for Parallelstore instance | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| instance | The created Parallelstore instance | +| instance_id | The ID of the Parallelstore instance | +| instance_name | The name of the Parallelstore instance | +| location | The location where the Parallelstore instance is created | + +## Notes + +- SCRATCH deployment type is optimized for temporary, high-performance workloads +- PERSISTENT deployment type is designed for long-term data storage with durability +- If no capacity is specified, defaults to 12000 GiB for SCRATCH and 27000 GiB for PERSISTENT +- Requires a properly configured VPC network with Private Service Access +- For best performance, ensure the network MTU is set to 8896 + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| capacity\_gib | Custom capacity in GiB for Parallelstore instance. If null, defaults to 12000 for SCRATCH and 27000 for PERSISTENT. | `number` | `null` | no | +| deployment\_type | Parallelstore Instance deployment type (SCRATCH or PERSISTENT) | `string` | `"SCRATCH"` | no | +| instance\_id | The ID of the Parallelstore instance. If null, will be set to 'parallelstore-{location}'. | `string` | `null` | no | +| location | The location (zone) where the Parallelstore instance will be created, in the format 'region-zone' e.g., 'us-central1-a' | `string` | `"null"` | no | +| network | The VPC network to which the Parallelstore instance should be connected | `string` | `"default"` | no | +| project\_id | The GCP project where the resources will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| access\_points | List of IPv4 addresses for the Parallelstore access points, required for client-side configuration including Kubernetes PersistentVolumes and Daemonsets | +| capacity\_gib | Provisioned storage capacity of the Parallelstore instance in Gibibytes (GiB), useful for capacity planning and cost estimation | +| daos\_version | Version of the Distributed Application Object Storage (DAOS) software running in the Parallelstore instance, useful for compatibility checks and documentation | +| id | Fully qualified identifier for the Parallelstore resource in format projects/{{project}}/locations/{{location}}/instances/{{instance\_id}}, used in API calls and scripts | +| instance\_id | ID of the Parallelstore instance | +| location | Zone location where the Parallelstore instance is deployed (format: {region}-{zone}), important for co-locating with compute resources | +| name | Fully qualified name of the Parallelstore instance in format projects/{{project}}/locations/{{location}}/instances/{{name}}, used in Google Cloud API calls | +| name\_short | Resource name in the format {{name}} | +| region | Region extracted from the location | +| reserved\_ip\_range | Identifier of the allocated IP address range associated with the Parallelstore private service access connection, used for network planning and troubleshooting | + + diff --git a/5-appinfra/modules/htc-infra/modules/parallelstore/main.tf b/5-appinfra/modules/htc-infra/modules/parallelstore/main.tf new file mode 100644 index 000000000..4af2beae9 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/parallelstore/main.tf @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + is_zone = can(regex("-[a-z]$", var.location)) + region = local.is_zone ? regex("^(.*)-[a-z]$", var.location)[0] : var.location + location = local.is_zone ? var.location : random_shuffle.zone.result[0] +} + +# data "google_project" "environment" { +# project_id = var.project_id +# } + +# Get available zones for the region +data "google_compute_zones" "available" { + project = var.project_id + region = local.region +} + +# Random zone selection +resource "random_shuffle" "zone" { + input = data.google_compute_zones.available.names + result_count = 1 +} + +# Create Parallelstore instance +resource "google_parallelstore_instance" "parallelstore" { + project = var.project_id + provider = google-beta + instance_id = var.instance_id != null ? var.instance_id : "parallelstore-${var.location}" + location = local.location + capacity_gib = var.capacity_gib != null ? var.capacity_gib : (var.deployment_type == "PERSISTENT" ? 27000 : 12000) + network = var.network + deployment_type = var.deployment_type + + timeouts { + create = "120m" + update = "120m" + delete = "120m" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/parallelstore/outputs.tf b/5-appinfra/modules/htc-infra/modules/parallelstore/outputs.tf new file mode 100644 index 000000000..8fdf31f81 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/parallelstore/outputs.tf @@ -0,0 +1,63 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "reserved_ip_range" { + description = "Identifier of the allocated IP address range associated with the Parallelstore private service access connection, used for network planning and troubleshooting" + value = google_parallelstore_instance.parallelstore.reserved_ip_range +} + +output "daos_version" { + description = "Version of the Distributed Application Object Storage (DAOS) software running in the Parallelstore instance, useful for compatibility checks and documentation" + value = google_parallelstore_instance.parallelstore.daos_version +} + +output "id" { + description = "Fully qualified identifier for the Parallelstore resource in format projects/{{project}}/locations/{{location}}/instances/{{instance_id}}, used in API calls and scripts" + value = google_parallelstore_instance.parallelstore.id +} + +output "name" { + description = "Fully qualified name of the Parallelstore instance in format projects/{{project}}/locations/{{location}}/instances/{{name}}, used in Google Cloud API calls" + value = google_parallelstore_instance.parallelstore.name +} + +output "instance_id" { + description = "ID of the Parallelstore instance" + value = google_parallelstore_instance.parallelstore.instance_id +} + +output "access_points" { + description = "List of IPv4 addresses for the Parallelstore access points, required for client-side configuration including Kubernetes PersistentVolumes and Daemonsets" + value = google_parallelstore_instance.parallelstore.access_points +} + +output "location" { + description = "Zone location where the Parallelstore instance is deployed (format: {region}-{zone}), important for co-locating with compute resources" + value = google_parallelstore_instance.parallelstore.location +} + +output "region" { + description = "Region extracted from the location" + value = split("-", google_parallelstore_instance.parallelstore.location)[0] +} + +output "capacity_gib" { + description = "Provisioned storage capacity of the Parallelstore instance in Gibibytes (GiB), useful for capacity planning and cost estimation" + value = google_parallelstore_instance.parallelstore.capacity_gib +} + +output "name_short" { + description = "Resource name in the format {{name}}" + value = var.instance_id != null ? var.instance_id : "parallelstore-${var.location}" +} diff --git a/5-appinfra/modules/htc-infra/modules/parallelstore/variables.tf b/5-appinfra/modules/htc-infra/modules/parallelstore/variables.tf new file mode 100644 index 000000000..f5e7b002c --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/parallelstore/variables.tf @@ -0,0 +1,58 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The GCP project where the resources will be created" + type = string + + validation { + condition = var.project_id != "YOUR_PROJECT_ID" + error_message = "'project_id' was not set, please set the value in the terraform.tfvars file" + } +} + +variable "location" { + description = "The location (zone) where the Parallelstore instance will be created, in the format 'region-zone' e.g., 'us-central1-a'" + type = string + default = "null" +} + +variable "instance_id" { + description = "The ID of the Parallelstore instance. If null, will be set to 'parallelstore-{location}'." + type = string + default = null +} + +variable "network" { + description = "The VPC network to which the Parallelstore instance should be connected" + type = string + default = "default" +} + +variable "deployment_type" { + description = "Parallelstore Instance deployment type (SCRATCH or PERSISTENT)" + type = string + default = "SCRATCH" + + validation { + condition = contains(["SCRATCH", "PERSISTENT"], var.deployment_type) + error_message = "The deployment_type must be either SCRATCH or PERSISTENT" + } +} + +variable "capacity_gib" { + description = "Custom capacity in GiB for Parallelstore instance. If null, defaults to 12000 for SCRATCH and 27000 for PERSISTENT." + type = number + default = null +} diff --git a/5-appinfra/modules/htc-infra/modules/parallelstore/versions.tf b/5-appinfra/modules/htc-infra/modules/parallelstore/versions.tf new file mode 100644 index 000000000..1eac1780d --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/parallelstore/versions.tf @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.4" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/project/README.md b/5-appinfra/modules/htc-infra/modules/project/README.md new file mode 100644 index 000000000..f680f76e8 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/project/README.md @@ -0,0 +1,98 @@ +# Google Cloud Project Setup Module + +This module configures a Google Cloud project with the necessary API services, IAM permissions, and settings required for risk and research workloads. + +## Usage + +```hcl +module "project_setup" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/project" + + project_id = "your-project-id" + enable_log_analytics = true +} +``` + +## Features + +- Enables required Google Cloud APIs for risk and research workloads +- Configures log analytics for enhanced observability +- Sets up IAM permissions for service accounts +- Enables API services for containers, storage, networking, and more +- Configures default project settings optimal for analytics workloads + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|----------| +| project_id | The GCP project where the resources will be created | `string` | n/a | yes | +| enable_log_analytics | Enable log analytics in the project | `bool` | `true` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| project_id | The ID of the configured project | +| project_number | The numeric project number | +| enabled_apis | List of enabled API services | + +## Enabled API Services + +The module enables the following API services: + +- artifactregistry.googleapis.com +- cloudresourcemanager.googleapis.com +- compute.googleapis.com +- container.googleapis.com +- containeranalysis.googleapis.com +- containerfilesystem.googleapis.com +- containerregistry.googleapis.com +- containersecurity.googleapis.com +- iam.googleapis.com +- iamcredentials.googleapis.com +- logging.googleapis.com +- monitoring.googleapis.com +- networksecurity.googleapis.com +- parallelstore.googleapis.com +- pubsub.googleapis.com +- servicenetworking.googleapis.com +- storage-api.googleapis.com +- storage.googleapis.com + +## Notes + +- Some API enablement may take time to propagate, and Terraform may need to be run multiple times +- When destroying the Terraform configuration, API services will not be disabled to prevent disruption to other services +- Log analytics configuration creates linked BigQuery datasets for log analysis + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| enable\_log\_analytics | Enable log analytics in the project? | `bool` | `true` | no | +| project\_id | The GCP project where the resources will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| analytics\_bigquery\_dataset\_id | ID of the BigQuery dataset linked to Cloud Logging for log analytics, returned only when log analytics is enabled, used for querying logs and creating dashboards | + + diff --git a/5-appinfra/modules/htc-infra/modules/project/main.tf b/5-appinfra/modules/htc-infra/modules/project/main.tf new file mode 100644 index 000000000..2ab9b38ff --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/project/main.tf @@ -0,0 +1,208 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_project" "environment" { + project_id = var.project_id +} + +resource "google_project_service" "artifactregistry_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "artifactregistry.googleapis.com" +} + +resource "google_project_service" "logging_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "logging.googleapis.com" +} + + +# resource "google_project_service" "anthos_googleapis_com" { +# disable_dependent_services = false +# disable_on_destroy = false +# project = data.google_project.environment.project_id +# service = "anthos.googleapis.com" +# } + +# resource "google_project_service" "anthosconfigmanagement_googleapis_com" { +# disable_dependent_services = false +# disable_on_destroy = false +# project = data.google_project.environment.project_id +# service = "anthosconfigmanagement.googleapis.com" +# } + +resource "google_project_service" "cloudresourcemanager_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "cloudresourcemanager.googleapis.com" +} + +resource "google_project_service" "compute_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "compute.googleapis.com" +} + +# resource "google_project_service" "connectgateway_googleapis_com" { +# disable_dependent_services = false +# disable_on_destroy = false +# project = data.google_project.environment.project_id +# service = "connectgateway.googleapis.com" +# } + +resource "google_project_service" "container_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "container.googleapis.com" +} + +resource "google_project_service" "containerfilesystem_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "containerfilesystem.googleapis.com" +} + +# resource "google_project_service" "containersecurity_googleapis_com" { +# disable_dependent_services = false +# disable_on_destroy = false +# project = data.google_project.environment.project_id +# service = "containersecurity.googleapis.com" +# } + +# resource "google_project_service" "gkeconnect_googleapis_com" { +# disable_dependent_services = false +# disable_on_destroy = false +# project = data.google_project.environment.project_id +# service = "gkeconnect.googleapis.com" +# } + +# resource "google_project_service" "gkehub_googleapis_com" { +# disable_dependent_services = false +# disable_on_destroy = false +# project = data.google_project.environment.project_id +# service = "gkehub.googleapis.com" +# } + +resource "google_project_service" "cloudquotas_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "cloudquotas.googleapis.com" +} + +resource "google_project_service" "iam_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "iam.googleapis.com" +} + +resource "google_project_service" "serviceusage_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "serviceusage.googleapis.com" +} + +# Vertex AI API's + +resource "google_project_service" "aiplatform_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "aiplatform.googleapis.com" +} + +resource "google_project_service" "notebooks_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "notebooks.googleapis.com" +} + +resource "google_project_service" "servicenetworking_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "servicenetworking.googleapis.com" +} + +resource "google_project_service" "parallelstore_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "parallelstore.googleapis.com" +} + +resource "google_project_service" "cloudkms_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "cloudkms.googleapis.com" +} + +resource "google_project_service" "cloudbuild_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.environment.project_id + service = "cloudbuild.googleapis.com" +} + +resource "google_project_service" "bigquery_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = var.project_id + service = "bigquery.googleapis.com" +} + +resource "google_project_service" "pubsub_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = var.project_id + service = "pubsub.googleapis.com" +} + +resource "google_project_service" "sts_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = var.project_id + service = "sts.googleapis.com" +} + +# Log Analytics + +resource "google_logging_project_bucket_config" "all_analytics_enabled_bucket" { + count = var.enable_log_analytics ? 1 : 0 + project = data.google_project.environment.project_id + location = "global" + enable_analytics = true + bucket_id = "_Default" + lifecycle { + ignore_changes = [project] + } +} + +resource "google_logging_linked_dataset" "all_logging_linked_dataset" { + count = var.enable_log_analytics ? 1 : 0 + link_id = "all_logging_bq_link" + bucket = google_logging_project_bucket_config.all_analytics_enabled_bucket[0].id + description = "Linked dataset" +} diff --git a/5-appinfra/modules/htc-infra/modules/project/outputs.tf b/5-appinfra/modules/htc-infra/modules/project/outputs.tf new file mode 100644 index 000000000..50e556d51 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/project/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "analytics_bigquery_dataset_id" { + description = "ID of the BigQuery dataset linked to Cloud Logging for log analytics, returned only when log analytics is enabled, used for querying logs and creating dashboards" + value = var.enable_log_analytics ? google_logging_linked_dataset.all_logging_linked_dataset[0].bigquery_dataset[0].dataset_id : null +} diff --git a/5-appinfra/modules/htc-infra/modules/project/variables.tf b/5-appinfra/modules/htc-infra/modules/project/variables.tf new file mode 100644 index 000000000..245b14845 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/project/variables.tf @@ -0,0 +1,29 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The GCP project where the resources will be created" + type = string + + validation { + condition = var.project_id != "YOUR_PROJECT_ID" + error_message = "'project_id' was not set, please set the value in the fsi-resaerch-1.tfvars file" + } +} + +variable "enable_log_analytics" { + description = "Enable log analytics in the project?" + type = bool + default = true +} diff --git a/5-appinfra/modules/htc-infra/modules/project/versions.tf b/5-appinfra/modules/htc-infra/modules/project/versions.tf new file mode 100644 index 000000000..d21257e22 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/project/versions.tf @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/README.md b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/README.md new file mode 100644 index 000000000..dbf26a273 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/README.md @@ -0,0 +1,85 @@ +# Google Cloud Pub/Sub to BigQuery Module + +This module creates Pub/Sub subscriptions that automatically write messages to BigQuery tables, enabling real-time data analytics for risk and research workloads. + +## Usage + +```hcl +module "pubsub_bigquery" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/pubsub-subscriptions" + + project_id = "your-project-id" + region = "us-central1" + bigquery_dataset = "analytics_dataset" + bigquery_table = "pubsub_messages" + topics = ["topic-1", "topic-2", "topic-3"] + subscriber_service_account = google_service_account.pubsub_sa.email +} +``` + +## Features + +- Creates BigQuery subscriptions for multiple Pub/Sub topics +- Automatically writes Pub/Sub messages to BigQuery tables +- Configures appropriate IAM permissions +- Supports schema validation for messages +- Enables real-time data analytics on streaming data + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|----------| +| project_id | The GCP project ID where resources will be created | `string` | n/a | yes | +| region | The Region of the Pub/Sub topic and subscriptions | `string` | n/a | yes | +| bigquery_dataset | The Dataset for capturing BigQuery data | `string` | n/a | yes | +| bigquery_table | The BigQuery table for capturing Pub/Sub data | `string` | n/a | yes | +| topics | List of topics to persist to BigQuery | `list(string)` | n/a | yes | +| subscriber_service_account | Service account that will be granted subscriber role on topics | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| subscriptions | List of created BigQuery subscriptions | +| subscription_paths | Full paths of the created subscriptions | + +## Notes + +- The module expects the BigQuery dataset and table to exist already +- Messages are written to BigQuery in near real-time +- Pub/Sub messages should be in JSON format for optimal BigQuery integration +- Schema validation can help ensure data quality +- The subscriber service account needs appropriate permissions to read from Pub/Sub and write to BigQuery + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| bigquery\_dataset | The Dataset for capturing BigQuery data | `string` | n/a | yes | +| bigquery\_table | The Datsaset for capturing BigQuery data | `string` | n/a | yes | +| project\_id | The GCP project ID where resources will be created. | `string` | n/a | yes | +| subscriber\_service\_account | Service account that will be granted subscriber role on topics | `string` | `""` | no | +| topics | List of topics to persist to BigQuery | `list(string)` | n/a | yes | + +## Outputs + +No outputs. + + diff --git a/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/main.tf b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/main.tf new file mode 100644 index 000000000..adfafcac3 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/main.tf @@ -0,0 +1,98 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# BigQuery persistence +# + +# resource "google_service_account" "bq_write_service_account" { +# project = var.project_id +# account_id = "pubsub-bigquery-writer" +# display_name = "BQ Write Service Account" +# } + +data "local_file" "pubsub_json_schema" { + filename = "${path.module}/pubsub_json_schema.txt" +} + +resource "google_bigquery_table" "messages" { + project = var.project_id + deletion_protection = false + table_id = var.bigquery_table + dataset_id = var.bigquery_dataset + + # 90 days + time_partitioning { + expiration_ms = 90 * 24 * 60 * 60 * 1000 + field = "publish_time" + type = "DAY" + } + + schema = data.local_file.pubsub_json_schema.content +} + +# Permission for metadata on the table +resource "google_bigquery_table_iam_member" "message_metadata" { + project = var.project_id + dataset_id = google_bigquery_table.messages.dataset_id + table_id = google_bigquery_table.messages.table_id + role = "roles/bigquery.metadataViewer" + # member = "serviceAccount:${google_service_account.bq_write_service_account.email}" + member = "serviceAccount:${var.subscriber_service_account}" +} + +# Permission for inserting into the table +resource "google_bigquery_table_iam_member" "message_editor" { + project = var.project_id + dataset_id = google_bigquery_table.messages.dataset_id + table_id = google_bigquery_table.messages.table_id + role = "roles/bigquery.dataEditor" + # member = "serviceAccount:${google_service_account.bq_write_service_account.email}" + member = "serviceAccount:${var.subscriber_service_account}" +} + + +# +# Want JSON-based PubSub subscriptions. +# + +# resource "google_pubsub_topic_iam_member" "topic_subscriber" { +# for_each = toset(var.topics) + +# project = var.project_id +# topic = each.value +# role = "roles/pubsub.subscriber" +# member = "serviceAccount:${google_service_account.bq_write_service_account.email}" +# } + +resource "google_pubsub_subscription" "bq_sub" { + for_each = toset(var.topics) + + depends_on = [ + google_bigquery_table_iam_member.message_editor, + google_bigquery_table_iam_member.message_metadata, + # google_pubsub_topic_iam_member.topic_subscriber, + ] + + project = var.project_id + name = "${each.value}_bq" + topic = each.value + + bigquery_config { + table = "${google_bigquery_table.messages.project}.${google_bigquery_table.messages.dataset_id}.${google_bigquery_table.messages.table_id}" + service_account_email = var.subscriber_service_account + write_metadata = true + } +} diff --git a/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/outputs.tf b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/outputs.tf new file mode 100644 index 000000000..d6b03081f --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/outputs.tf @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/pubsub_json_schema.txt b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/pubsub_json_schema.txt new file mode 100644 index 000000000..51b365fac --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/pubsub_json_schema.txt @@ -0,0 +1,7 @@ +[ +{ "name": "subscription_name", "type": "STRING", "mode": "NULLABLE", "description": "Subscription name" }, +{ "name": "message_id", "type": "STRING", "mode": "NULLABLE", "description": "Message ID" }, +{ "name": "publish_time", "type": "TIMESTAMP", "mode": "NULLABLE", "description": "Publish time" }, +{ "name": "attributes", "type": "JSON", "mode": "NULLABLE", "description": "Message attributes" }, +{ "name": "data", "type": "JSON", "mode": "NULLABLE", "description": "JSON message" } +] diff --git a/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/variables.tf b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/variables.tf new file mode 100644 index 000000000..f75e241cc --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/variables.tf @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Project ID where resources will be deployed +variable "project_id" { + type = string + description = "The GCP project ID where resources will be created." +} + +# Region where the build and artifact repository is +# variable "region" { +# type = string +# description = "The Region of the Pub/Sub topic and subscriptions" +# } + +# Region where the build and artifact repository is +variable "bigquery_dataset" { + type = string + description = "The Dataset for capturing BigQuery data" +} + +# Region where the build and artifact repository is +variable "bigquery_table" { + type = string + description = "The Datsaset for capturing BigQuery data" +} + +# Containers to build +variable "topics" { + type = list(string) + description = "List of topics to persist to BigQuery" +} + +variable "subscriber_service_account" { + description = "Service account that will be granted subscriber role on topics" + type = string + default = "" +} diff --git a/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/versions.tf b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/versions.tf new file mode 100644 index 000000000..13b507f25 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/versions.tf @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.4" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + local = { + source = "hashicorp/local" + version = ">= 2.5.3" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/quota/README.md b/5-appinfra/modules/htc-infra/modules/quota/README.md new file mode 100644 index 000000000..769bdefdb --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/quota/README.md @@ -0,0 +1,116 @@ +# Google Cloud Quota Preferences Module + +This module allows you to request multiple quota preferences for your Google Cloud project using the `google_cloud_quotas_quota_preference` resource. It provides a flexible way to request various quota increases across different Google Cloud services and regions. + +## Usage + +```hcl +module "quota" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/quota" + + project_id = "your-project-id" + quota_contact_email = "your-email@example.com" + + quota_preferences = [ + { + service = "compute.googleapis.com" + quota_id = "PREEMPTIBLE-CPUS-per-project-region" + preferred_value = 10000 + region = "us-central1" + }, + { + service = "compute.googleapis.com" + quota_id = "DISKS-TOTAL-GB-per-project-region" + preferred_value = 65000 + region = "us-central1" + }, + { + service = "monitoring.googleapis.com" + quota_id = "IngestionRequestsPerMinutePerProject" + preferred_value = 100000 + } + ] +} +``` + +## Features + +- Request multiple quota preferences in one module +- Automatically handle region-specific quotas +- Skip quota requests when no contact email is provided +- Customizable quota preference names + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|----------| +| project_id | The GCP project where the resources will be created | `string` | n/a | yes | +| region | The default region to host resources in | `string` | `"us-central1"` | no | +| quota_contact_email | Contact email for quota requests | `string` | `""` | no | +| quota_preferences | List of quota preferences to request | `list(object)` | `[]` | no | + +### Quota Preferences Object Structure + +```hcl +object({ + service = string # Service name (e.g., "compute.googleapis.com") + quota_id = string # Quota ID (e.g., "PREEMPTIBLE-CPUS-per-project-region") + preferred_value = number # Requested quota value + dimensions = map(string) # Optional dimensions (will include region if specified) + region = string # Optional region (will be added to dimensions) + custom_name = string # Optional custom name for the resource +}) +``` + +## Outputs + +| Name | Description | +|------|-------------| +| quota_preferences | Map of created quota preferences | +| requested_quota_count | Number of quota preferences requested | + +## Examples + +An example implementation can be found in the [examples/infrastructure/multi-quota](../../examples/infrastructure/multi-quota) directory, demonstrating how to request quotas for multiple regions and services in a single module call. + +## Notes + +- If `quota_contact_email` is empty, no quota preferences will be created +- The module uses a lifecycle configuration to ignore all changes after creation, as quota updates are handled by Google Cloud +- For region-specific quotas, provide the region in the `region` field and it will be properly added to dimensions +- This module sends quota **requests** that must be approved by Google Cloud. Check the [Quotas documentation](https://cloud.google.com/docs/quotas) for more information +- Quota requests may take time to process and are not guaranteed to be approved + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The GCP project where the resources will be created | `string` | n/a | yes | +| quota\_contact\_email | Contact email for quota requests | `string` | `""` | no | +| quota\_preferences | Map of quota preferences to request. Each item should include service, quota\_id, preferred\_value, and optional dimensions | 
list(object({
    service         = string
    quota_id        = string
    preferred_value = number
    dimensions      = optional(map(string), {})
    region          = optional(string, null)
    # Use custom_name if you want to override the default name format
    custom_name = optional(string, null)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| quota\_preferences | Map of created quota preferences | +| requested\_quota\_count | Number of quota preferences requested | + + diff --git a/5-appinfra/modules/htc-infra/modules/quota/main.tf b/5-appinfra/modules/htc-infra/modules/quota/main.tf new file mode 100644 index 000000000..93abf3f2b --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/quota/main.tf @@ -0,0 +1,51 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # Process quota preferences to handle region-specific quotas + quota_preferences = [ + for pref in var.quota_preferences : { + service = pref.service + quota_id = pref.quota_id + preferred_value = pref.preferred_value + # Merge region into dimensions if present and not already in dimensions + dimensions = pref.region != null ? merge(pref.dimensions, { region = pref.region }) : pref.dimensions + # Generate name if custom_name is not provided + name = pref.custom_name != null ? pref.custom_name : "${replace(pref.service, ".", "_")}-${pref.quota_id}${pref.region != null ? "_${pref.region}" : ""}" + } + ] +} + +# data "google_project" "environment" { +# project_id = var.project_id +# } + +resource "google_cloud_quotas_quota_preference" "quota_preferences" { + for_each = { for idx, pref in local.quota_preferences : idx => pref if var.quota_contact_email != "" } + + parent = "projects/${var.project_id}" + name = each.value.name + dimensions = each.value.dimensions + service = each.value.service + quota_id = each.value.quota_id + contact_email = var.quota_contact_email + + quota_config { + preferred_value = each.value.preferred_value + } + + lifecycle { + ignore_changes = all + } +} diff --git a/5-appinfra/modules/htc-infra/modules/quota/outputs.tf b/5-appinfra/modules/htc-infra/modules/quota/outputs.tf new file mode 100644 index 000000000..7cd70ba46 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/quota/outputs.tf @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "quota_preferences" { + description = "Map of created quota preferences" + value = google_cloud_quotas_quota_preference.quota_preferences +} + +output "requested_quota_count" { + description = "Number of quota preferences requested" + value = length(google_cloud_quotas_quota_preference.quota_preferences) +} diff --git a/5-appinfra/modules/htc-infra/modules/quota/variables.tf b/5-appinfra/modules/htc-infra/modules/quota/variables.tf new file mode 100644 index 000000000..b68789e23 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/quota/variables.tf @@ -0,0 +1,59 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The GCP project where the resources will be created" + type = string + + validation { + condition = var.project_id != "YOUR_PROJECT_ID" + error_message = "'project_id' was not set, please set the value in your terraform.tfvars file" + } +} + +# variable "region" { +# description = "The region to host the cluster in" +# type = string +# default = "us-central1" +# } + +variable "quota_contact_email" { + description = "Contact email for quota requests" + type = string + default = "" +} + +variable "quota_preferences" { + description = "Map of quota preferences to request. Each item should include service, quota_id, preferred_value, and optional dimensions" + type = list(object({ + service = string + quota_id = string + preferred_value = number + dimensions = optional(map(string), {}) + region = optional(string, null) + # Use custom_name if you want to override the default name format + custom_name = optional(string, null) + })) + default = [] + + validation { + condition = length([for q in var.quota_preferences : q if q.service == ""]) == 0 + error_message = "The 'service' field must be provided for each quota preference." + } + + validation { + condition = length([for q in var.quota_preferences : q if q.quota_id == ""]) == 0 + error_message = "The 'quota_id' field must be provided for each quota preference." + } +} diff --git a/5-appinfra/modules/htc-infra/modules/quota/versions.tf b/5-appinfra/modules/htc-infra/modules/quota/versions.tf new file mode 100644 index 000000000..d21257e22 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/quota/versions.tf @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/modules/region-analysis/README.md b/5-appinfra/modules/htc-infra/modules/region-analysis/README.md new file mode 100644 index 000000000..1d59960e9 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/region-analysis/README.md @@ -0,0 +1,99 @@ +# Region Analysis Module + +This module analyzes a list of GCP regions to determine the optimal multi-region or default region for deploying resources across multiple geographies. + +## Usage + +```hcl +module "region_analysis" { + source = "github.com/GoogleCloudPlatform/risk-and-research-blueprints//terraform/modules/region-analysis" + + regions = ["us-central1", "us-east4", "europe-west4"] +} + +# Use the determined multi-region +resource "google_storage_bucket" "multi_region_bucket" { + name = "my-bucket" + location = module.region_analysis.multi_region + force_destroy = true +} + +# Use the default region when needed +resource "google_pubsub_topic" "topic" { + name = "my-topic" + project = "my-project" + location = module.region_analysis.default_region +} +``` + +## Features + +- Determines the optimal multi-region (us, europe, asia) based on region distribution +- Provides a default region for resources that must be in a single region +- Handles tie-breaking with a preference for US > Europe > Asia +- Maps individual regions to their corresponding multi-region parent + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|----------| +| regions | List of regions to determine multi region choice | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| multi_region | The optimal multi-region based on the provided regions | +| default_region | The first region in the provided list, to be used as a default | + +## Algorithm + +The module determines the optimal multi-region using the following logic: + +1. Map each region to its parent multi-region (us, europe, asia) +2. Count the number of regions in each multi-region +3. Select the multi-region with the highest count +4. In case of ties, use a preference order: US > Europe > Asia +5. If no regions match known patterns, default to "us" + +## Notes + +- This module is useful for determining optimal locations for global resources +- The multi-region output is ideal for storage buckets and other multi-region resources +- The default_region output is useful for services that must exist in a single region +- This approach ensures that resources are created in regions that are geographically close to your compute resources + +## License + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| regions | List of regions to determine multi region choice | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| default\_region | The first alphabetically sorted individual region from within the dominant multi-region, used as a default region for resources requiring a specific region | +| dominant\_region | The multi-region with the most regions in the input set (preferring US in case of ties), used for optimal placement of multi-region resources | +| region\_counts | Count of specific regions within each multi-region group, used to determine the dominant multi-region and for resource distribution planning | +| region\_mapping | Mapping of individual GCP regions to their corresponding multi-region parent (e.g., us-central1 → us), used for determining resource placement strategy | +| regions\_in\_dominant | List of all individual regions that belong to the identified dominant multi-region, useful for distributing resources across zones within the preferred multi-region | + + diff --git a/5-appinfra/modules/htc-infra/modules/region-analysis/main.tf b/5-appinfra/modules/htc-infra/modules/region-analysis/main.tf new file mode 100644 index 000000000..fb88b857a --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/region-analysis/main.tf @@ -0,0 +1,58 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # Map regions to their multi-region parent + region_mapping = { + for region in var.regions : region => ( + can(regex("^us-", region)) ? "us" : + can(regex("^europe-", region)) ? "europe" : + can(regex("^asia-", region)) ? "asia" : "unknown" + ) + } + + # Count occurrences of each multi-region + region_counts = { + us = length([for region, multi in local.region_mapping : region if multi == "us"]) + europe = length([for region, multi in local.region_mapping : region if multi == "europe"]) + asia = length([for region, multi in local.region_mapping : region if multi == "asia"]) + } + + # Find the maximum count + max_count = max(values(local.region_counts)...) + + # Create a map of multi-regions at max count + max_count_regions = { + for name, count in local.region_counts : name => count + if count == local.max_count + } + + # Determine dominant region with US preference + dominant_region = contains(keys(local.max_count_regions), "us") ? "us" : ( + length(keys(local.max_count_regions)) > 0 ? sort(keys(local.max_count_regions))[0] : "us" + ) + + # Get all regions from the dominant multi-region + regions_in_dominant = [ + for region, multi in local.region_mapping : region + if multi == local.dominant_region + ] + + # Sort all input regions to ensure deterministic selection for fallback + sorted_input_regions = sort(var.regions) + + # Select the alphabetically first region from the dominant multi-region + # If no regions in dominant, fall back to first region from input list + default_region = length(local.regions_in_dominant) > 0 ? sort(local.regions_in_dominant)[0] : local.sorted_input_regions[0] +} diff --git a/5-appinfra/modules/htc-infra/modules/region-analysis/output.tf b/5-appinfra/modules/htc-infra/modules/region-analysis/output.tf new file mode 100644 index 000000000..bf281d6dc --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/region-analysis/output.tf @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +output "region_mapping" { + description = "Mapping of individual GCP regions to their corresponding multi-region parent (e.g., us-central1 → us), used for determining resource placement strategy" + value = local.region_mapping +} + +output "region_counts" { + description = "Count of specific regions within each multi-region group, used to determine the dominant multi-region and for resource distribution planning" + value = local.region_counts +} + +output "dominant_region" { + description = "The multi-region with the most regions in the input set (preferring US in case of ties), used for optimal placement of multi-region resources" + value = local.dominant_region +} + +output "default_region" { + description = "The first alphabetically sorted individual region from within the dominant multi-region, used as a default region for resources requiring a specific region" + value = local.default_region +} + +output "regions_in_dominant" { + description = "List of all individual regions that belong to the identified dominant multi-region, useful for distributing resources across zones within the preferred multi-region" + value = local.regions_in_dominant +} diff --git a/5-appinfra/modules/htc-infra/modules/region-analysis/variables.tf b/5-appinfra/modules/htc-infra/modules/region-analysis/variables.tf new file mode 100644 index 000000000..657c4094c --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/region-analysis/variables.tf @@ -0,0 +1,19 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Region for resource deployment (default: us-central1) +variable "regions" { + description = "List of regions to determine multi region choice" + type = list(string) +} diff --git a/5-appinfra/modules/htc-infra/modules/region-analysis/versions.tf b/5-appinfra/modules/htc-infra/modules/region-analysis/versions.tf new file mode 100644 index 000000000..d21257e22 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/region-analysis/versions.tf @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/outputs.tf b/5-appinfra/modules/htc-infra/outputs.tf new file mode 100644 index 000000000..024ed00b1 --- /dev/null +++ b/5-appinfra/modules/htc-infra/outputs.tf @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "ui_config" { + description = "Yaml configuration for UI deployment" + value = local.ui_config_file +} + +output "test_scripts" { + description = "Test configuration shell scripts" + value = module.gke.test_scripts_list +} diff --git a/5-appinfra/modules/htc-infra/sql/README.md b/5-appinfra/modules/htc-infra/sql/README.md new file mode 100644 index 000000000..ce40dd9f5 --- /dev/null +++ b/5-appinfra/modules/htc-infra/sql/README.md @@ -0,0 +1,21 @@ + +Interesting query: + +SELECT + start_time, + jobId, + initId, + TIMESTAMP_DIFF(last_process_time, first_process_time, SECOND) as run_secs, + last_process_time AS last_time, + remaining_tasks AS queue, + CAST(SAFE_DIVIDE(100*task_jobs,remaining_tasks+task_jobs) AS STRING FORMAT '999.9') AS perc_done, + init_jobs, + CAST(SAFE_DIVIDE(task_compute_secs, task_jobs) AS STRING FORMAT '999999.9') AS init_cpu_avg, + CAST(SAFE_DIVIDE(init_io_secs_total, init_jobs) AS STRING FORMAT '999999.9') AS init_io_avg, + task_jobs, + CAST(SAFE_DIVIDE(task_compute_secs, task_jobs) AS STRING FORMAT '999999.9') AS task_cpu_avg, + CAST(SAFE_DIVIDE(task_io_secs_total, task_jobs) AS STRING FORMAT '999999.9') AS task_io_avg, +from + `fsi-scratch-18.workload.messages_summary` +ORDER BY + start_time DESC diff --git a/5-appinfra/modules/htc-infra/sql/agent_stats.sql b/5-appinfra/modules/htc-infra/sql/agent_stats.sql new file mode 100644 index 000000000..581aaa207 --- /dev/null +++ b/5-appinfra/modules/htc-infra/sql/agent_stats.sql @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +SELECT + timestamp, + meta, + INT64(JSON_QUERY(json_payload, "$.bytes")) AS bytes, + INT64(JSON_QUERY(json_payload, "$.ops")) as ops, + FLOAT64(JSON_QUERY(json_payload, "$.load")) as load, + INT64(JSON_QUERY(json_payload, "$.lat_min")) AS lat_min, + INT64(JSON_QUERY(json_payload, "$.lat_50")) AS lat_50, + INT64(JSON_QUERY(json_payload, "$.lat_95")) AS lat_95, + INT64(JSON_QUERY(json_payload, "$.lat_99")) AS lat_99, + INT64(JSON_QUERY(json_payload, "$.lat_max")) AS lat_max +FROM + `${project_id}.${dataset_id}.${table_id}` +WHERE + meta.job_container_id IN ('agent', 'controller') AND + meta.job_worker_type IS NOT NULL AND + JSON_VALUE(json_payload, "$.msg") = 'statistics' diff --git a/5-appinfra/modules/htc-infra/sql/agent_summary_by_instance.sql b/5-appinfra/modules/htc-infra/sql/agent_summary_by_instance.sql new file mode 100644 index 000000000..4c303310d --- /dev/null +++ b/5-appinfra/modules/htc-infra/sql/agent_summary_by_instance.sql @@ -0,0 +1,59 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +WITH + SrcData AS ( + SELECT + meta.infra, + meta.job_worker_type, + meta.job_worker_id, + meta.job_container_id, + timestamp, + (timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 60 SECOND)) AS win_1min, + (timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 5*60 SECOND)) AS win_5min, + (timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 60*60 SECOND)) AS win_15min, + bytes, + ops, + load + FROM + `${project_id}.${dataset_id}.${table_id}` + ) +SELECT + infra, + job_worker_type, + job_worker_id, + job_container_id, + MAX(timestamp) AS last_active_time, + TIMESTAMP_DIFF(MAX(timestamp), MIN(timestamp), SECOND) AS total_active_secs, + SAFE_DIVIDE(SUM(IF(win_1min,ops,0)),60) AS win_1min_ops_per_second, + SAFE_DIVIDE(SUM(IF(win_1min,bytes,0)),60) AS win_1min_bytes_per_second, + SAFE_DIVIDE(SUM(IF(win_1min,load,0)),SUM(IF(win_1min,1,0))) AS win_1min_load_per_second, + SAFE_DIVIDE(SUM(IF(win_5min,ops,0)),300) AS win_5min_ops_per_second, + SAFE_DIVIDE(SUM(IF(win_5min,bytes,0)),300) AS win_5min_bytes_per_second, + SAFE_DIVIDE(SUM(IF(win_5min,load,0)),SUM(IF(win_5min,1,0))) AS win_5min_load_per_second, + SAFE_DIVIDE(SUM(IF(win_15min,ops,0)),900) AS win_15min_ops_per_second, + SAFE_DIVIDE(SUM(IF(win_15min,bytes,0)),900) AS win_15min_bytes_per_second, + SAFE_DIVIDE(SUM(IF(win_15min,load,0)),SUM(IF(win_15min,1,0))) AS win_15min_load_per_second, + SAFE_DIVIDE(SUM(ops),TIMESTAMP_DIFF(MAX(timestamp), MIN(timestamp), SECOND)) AS life_ops_per_second, + SAFE_DIVIDE(SUM(bytes),TIMESTAMP_DIFF(MAX(timestamp), MIN(timestamp), SECOND)) AS life_bytes_per_second, + SAFE_DIVIDE(SUM(load),SUM(1)) AS life_load_per_second +FROM + SrcData +GROUP BY + infra, + job_worker_type, + job_worker_id, + job_container_id diff --git a/5-appinfra/modules/htc-infra/sql/log_stats.sql b/5-appinfra/modules/htc-infra/sql/log_stats.sql new file mode 100644 index 000000000..553018537 --- /dev/null +++ b/5-appinfra/modules/htc-infra/sql/log_stats.sql @@ -0,0 +1,39 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +SELECT + timestamp, + STRUCT( + resource.type AS infra, + IF(resource.type='cloud_run_job', + JSON_VALUE(resource.labels, "$.job_name"), + IF(resource.type='cloud_run_revision', + JSON_VALUE(resource.labels, "$.service_name"), + JSON_VALUE(labels, "$.\"k8s-pod/app\"")) + ) AS job_worker_type, + IF(resource.type IN ('cloud_run_job', 'cloud_run_revision'), + JSON_VALUE(labels, "$.instanceId"), + JSON_VALUE(resource.labels, "$.pod_name")) AS job_worker_id, + IF(resource.type IN ('cloud_run_job', 'cloud_run_revision'), + COALESCE(JSON_VALUE(labels, "$.container_name"), JSON_VALUE(resource.labels, "$.job_name")), + JSON_VALUE(resource.labels, "$.container_name")) AS job_container_id + ) AS meta, + * EXCEPT (timestamp) +FROM + `${project_id}.${dataset_id}._AllLogs` +WHERE + resource.type IN ('k8s_container', 'cloud_run_job', 'cloud_run_revision') AND + json_payload IS NOT NULL diff --git a/5-appinfra/modules/htc-infra/sql/pubsub_messages_joined.sql b/5-appinfra/modules/htc-infra/sql/pubsub_messages_joined.sql new file mode 100644 index 000000000..ab5f9e479 --- /dev/null +++ b/5-appinfra/modules/htc-infra/sql/pubsub_messages_joined.sql @@ -0,0 +1,100 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +WITH + Src AS ( + SELECT + JSON_VALUE(attributes, "$.srcId") AS msgId, + REGEXP_EXTRACT(subscription_name, r"^projects/[0-9]+/subscriptions/(.*)$") AS subscription_name, + message_id, + publish_time, + SAFE_CAST(JSON_VALUE(attributes, "$.srcTimeNano") AS INT64) AS srcTimeNano, + JSON_VALUE(attributes, "$.Hostname") AS hostname, + data + FROM + `${project_id}.${dataset_id}.${table_id}` + ), + Request AS ( + SELECT + msgId, + + subscription_name AS request_subscription_name, + message_id AS request_message_id, + publish_time AS request_time, + + LAX_INT64(JSON_QUERY(data, "$.init.id")) AS request_initId, + LAX_FLOAT64(JSON_QUERY(data, "$.init.percCrash")) AS request_initPercCrash, + LAX_FLOAT64(JSON_QUERY(data, "$.init.percFail")) AS request_initPercFail, + LAX_INT64(JSON_QUERY(data, "$.init.maxMicros")) AS request_initMaxMicros, + LAX_INT64(JSON_QUERY(data, "$.init.minMicros")) AS request_initMinMicros, + LAX_INT64(JSON_QUERY(data, "$.init.resultSize")) AS request_initResultSize, + LENGTH(JSON_VALUE(data, "$.init.payload")) AS request_initPayloadSize, + JSON_VALUE(data, "$.init.readFile") AS request_initReadFile, + JSON_VALUE(data, "$.init.readDir") AS request_initReadDir, + JSON_VALUE(data, "$.init.writeFile") AS request_initWriteFile, + LAX_INT64(JSON_QUERY(data, "$.init.writeBytes")) AS request_initWriteBytes, + + LAX_INT64(JSON_QUERY(data, "$.task.id")) AS request_taskId, + LAX_FLOAT64(JSON_QUERY(data, "$.task.percCrash")) AS request_taskPercCrash, + LAX_FLOAT64(JSON_QUERY(data, "$.task.percFail")) AS request_taskPercFail, + LAX_INT64(JSON_QUERY(data, "$.task.maxMicros")) AS request_taskMaxMicros, + LAX_INT64(JSON_QUERY(data, "$.task.minMicros")) AS request_taskMinMicros, + LAX_INT64(JSON_QUERY(data, "$.task.resultSize")) AS request_taskResultSize, + LENGTH(JSON_VALUE(data, "$.task.payload")) AS request_taskPayloadSize, + JSON_VALUE(data, "$.task.readFile") AS request_taskReadFile, + JSON_VALUE(data, "$.task.readDir") AS request_taskReadDir, + JSON_VALUE(data, "$.task.writeFile") AS request_taskWriteFile, + LAX_INT64(JSON_QUERY(data, "$.task.writeBytes")) AS request_taskWriteBytes + FROM + Src s + WHERE + hostname IS NULL + ), + Response AS ( + SELECT + msgId, + + subscription_name AS response_subscription_name, + message_id AS response_message_id, + publish_time AS response_time, + hostname, + + LAX_INT64(JSON_QUERY(data, "$.init.id")) AS response_initId, + LENGTH(JSON_VALUE(data, "$.init.payload")) AS response_initPayloadSize, + LAX_INT64(JSON_QUERY(data, "$.init.compute_micros")) AS response_initComputeMicros, + LAX_INT64(JSON_QUERY(data, "$.init.files_read")) AS response_initFilesRead, + LAX_INT64(JSON_QUERY(data, "$.init.bytes_read")) AS response_initBytesRead, + LAX_INT64(JSON_QUERY(data, "$.init.bytes_written")) AS response_initBytesWritten, + LAX_INT64(JSON_QUERY(data, "$.init.total_micros")) AS response_initTotalMicros, + + LAX_INT64(JSON_QUERY(data, "$.task.id")) AS response_taskId, + LENGTH(JSON_VALUE(data, "$.task.payload")) AS response_taskPayloadSize, + LAX_INT64(JSON_QUERY(data, "$.task.compute_micros")) AS response_taskComputeMicros, + LAX_INT64(JSON_QUERY(data, "$.task.files_read")) AS response_taskFilesRead, + LAX_INT64(JSON_QUERY(data, "$.task.bytes_read")) AS response_taskBytesRead, + LAX_INT64(JSON_QUERY(data, "$.task.bytes_written")) AS response_taskBytesWritten, + LAX_INT64(JSON_QUERY(data, "$.task.total_micros")) AS response_taskTotalMicros + FROM + Src + WHERE + hostname IS NOT NULL + ) +SELECT + REGEXP_EXTRACT(msgId, "Job-([0-9]+)-") AS jobId, + * +FROM + Request + LEFT JOIN Response USING (msgId); diff --git a/5-appinfra/modules/htc-infra/sql/pubsub_messages_summary.sql b/5-appinfra/modules/htc-infra/sql/pubsub_messages_summary.sql new file mode 100644 index 000000000..0e7b371fa --- /dev/null +++ b/5-appinfra/modules/htc-infra/sql/pubsub_messages_summary.sql @@ -0,0 +1,73 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +WITH + SrcData AS ( + SELECT + request_time, + response_time, + jobId, + msgId, + hostname, + request_initId AS initId, + response_initBytesRead/(1024*1024) AS init_mb_read, + response_initFilesRead as init_files_read, + (response_initTotalMicros - response_initComputeMicros)/1e6 as init_io_secs, + response_initComputeMicros/1e6 AS init_compute_secs, + response_taskBytesRead/(1024*1024) as task_mb_read, + response_taskBytesWritten/(1024*1024) as task_mb_written, + (response_taskTotalMicros - response_taskComputeMicros)/1e6 AS task_io_secs, + response_taskComputeMicros/1e6 AS task_compute_secs + FROM + `${project_id}.${dataset_id}.${joined_table_id}` + ), + SumStats AS ( + SELECT + jobId, + initId, + + -- Job metrics and timings + MIN(request_time) AS start_time, + MIN(response_time) AS first_process_time, + MAX(response_time) AS last_process_time, + COUNTIF(task_compute_secs IS NULL) AS remaining_tasks, + + -- Init stats + COUNTIF(init_compute_secs>0) AS init_jobs, + SUM(init_compute_secs) AS init_compute_secs, + MAX(init_io_secs) AS init_io_secs_max, + MIN(IF(init_io_secs>0,init_io_secs,999999999)) AS init_io_secs_min, + SUM(init_io_secs) AS init_io_secs_total, + SUM(init_mb_read) AS init_mb_read_total, + + -- Task stats + COUNTIF(task_compute_secs>0) AS task_jobs, + SUM(task_compute_secs) AS task_compute_secs, + MAX(task_io_secs) AS task_io_secs_max, + MIN(IF(task_io_secs>0, task_io_secs,999999999)) AS task_io_secs_min, + SUM(task_io_secs) AS task_io_secs_total, + SUM(task_mb_read) AS task_mb_read_total, + + FROM + SrcData + GROUP BY + JobId, + initId + ) +SELECT + * +FROM + SumStats; diff --git a/5-appinfra/modules/htc-infra/variables.tf b/5-appinfra/modules/htc-infra/variables.tf new file mode 100644 index 000000000..a24f0f897 --- /dev/null +++ b/5-appinfra/modules/htc-infra/variables.tf @@ -0,0 +1,248 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------- +# Project and Regional Configuration +#----------------------------------------------------- +variable "regions" { + description = "List of regions where GKE clusters should be created. Used for multi-region deployments." + type = list(string) + default = ["us-central1"] + + validation { + condition = length(var.regions) <= 4 + error_message = "Maximum 4 regions supported" + } +} + +#----------------------------------------------------- +# PubSub Configuration +#----------------------------------------------------- + +variable "pubsub_exactly_once" { + description = "Enable Pub/Sub exactly once subscriptions" + type = bool + default = true +} + +# variable "request_topic" { +# description = "Request topic for tasks" +# type = string +# default = "request" +# } + +# variable "request_subscription" { +# description = "Request subscription for tasks" +# type = string +# default = "request_sub" +# } + +# variable "response_topic" { +# description = "Response topic for tasks" +# type = string +# default = "response" +# } + +# variable "response_subscription" { +# description = "Response subscription for tasks" +# type = string +# default = "response_sub" +# } + +#----------------------------------------------------- +# BigQuery Configuration +#----------------------------------------------------- + +# variable "dataset_id" { +# description = "BigQuery dataset in the project to create the tables" +# type = string +# default = "pubsub_msgs" +# } + +#----------------------------------------------------- +# Quota Configuration +#----------------------------------------------------- + +variable "additional_quota_enabled" { + description = "Enable quota requests for additional resources" + type = bool + default = false +} + +variable "quota_contact_email" { + description = "Contact email for quota requests" + type = string + default = "" +} + +# variable "scaled_control_plane" { +# description = "Deploy a larger initial nodepool to ensure larger control plane nodes are provisioned" +# type = bool +# default = false +# } + +# variable "cluster_max_cpus" { +# description = "Maximum CPU cores in cluster autoscaling resource limits" +# type = number +# default = 10000 +# } + +# variable "cluster_max_memory" { +# description = "Maximum memory (in GB) in cluster autoscaling resource limits" +# type = number +# default = 80000 +# } + +#----------------------------------------------------- +# Storage Configuration +#----------------------------------------------------- + +variable "storage_type" { + description = "The type of storage system to deploy (PARALLELSTORE, LUSTRE, or null for none)" + type = string + default = null + + # validation { + # condition = var.storage_type == null || contains(["PARALLELSTORE", "LUSTRE"], var.storage_type) + # error_message = "The storage_type must be null, PARALLELSTORE, or LUSTRE." + # } +} + +variable "storage_capacity_gib" { + description = "Capacity in GiB for the selected storage system (Parallelstore or Lustre)" + type = number + default = null + + # validation { + # condition = var.storage_capacity_gib > 0 + # error_message = "Storage capacity must be a positive number or null." + # } +} + +variable "storage_locations" { + description = "Map of region to location (zone) for storage instances e.g. {\"us-central1\" = \"us-central1-a\"}" + type = map(string) + default = {} +} + + +#----------------------------------------------------- +# Storage Options +#----------------------------------------------------- + +variable "hsn_bucket" { + description = "Enable hierarchical namespace GCS buckets" + type = bool + default = false +} + +#----------------------------------------------------- +# VPC-SC +#----------------------------------------------------- + +variable "service_perimeter_name" { + description = "(VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter." + type = string + default = null +} + +variable "service_perimeter_mode" { + description = "(VPC-SC) Service perimeter mode: ENFORCE, DRY_RUN." + type = string + default = "DRY_RUN" + + validation { + condition = contains(["ENFORCE", "DRY_RUN"], var.service_perimeter_mode) + error_message = "The service_perimeter_mode value must be one of: ENFORCE, DRY_RUN." + } +} + +variable "access_level_name" { + description = "(VPC-SC) Access Level full name. When providing this variable, additional identities will be added to the access level, these are required to work within an enforced VPC-SC Perimeter." + type = string + default = null +} + +variable "infra_project" { + description = "The infrastructure project where resources will be managed." + type = string +} + +variable "admin_project" { + description = "The admin project where cloudbuild/cloudrun configurations will be managed." + type = string +} + +variable "service_name" { + type = string + description = "service name (e.g. 'transactionhistory')" +} + +variable "region" { + description = "The region where the cloud resources will be deployed." + type = string +} + + +variable "network_name" { + description = "VPC Network Name" + type = string +} + +variable "network_self_link" { + description = "VPC Network self link" + type = string +} + +variable "gke_cluster_names" { + description = "GKE Cluster Name to be used in configurations" + type = list(string) +} + +variable "parallelstore_deployment_type" { + description = "Parallelstore Instance deployment type (SCRATCH or PERSISTENT)" + type = string + default = "SCRATCH" + + validation { + condition = contains(["SCRATCH", "PERSISTENT"], var.parallelstore_deployment_type) + error_message = "deployment_type must be either SCRATCH or PERSISTENT" + } +} + +variable "cluster_project_id" { + type = string + description = "The GCP project ID where the cluster is created." +} + +variable "cluster_project_number" { + type = string + description = "The GCP project ID where the cluster is created." +} + +variable "team" { + description = "Environment Team, must be the same as the fleet scope team" + type = string +} + +variable "env" { + description = "The environment to prepare (ex. development)" + type = string +} + +variable "enable_csi_parallelstore" { + description = "Enable the Parallelstore CSI Driver" + type = bool + default = true +} diff --git a/5-appinfra/modules/htc-infra/versions.tf b/5-appinfra/modules/htc-infra/versions.tf new file mode 100644 index 000000000..fc108a017 --- /dev/null +++ b/5-appinfra/modules/htc-infra/versions.tf @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.5.3" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.4" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-loadtest-v1.0.0" + } +} diff --git a/5-appinfra/modules/htc-infra/vpc-sc.tf b/5-appinfra/modules/htc-infra/vpc-sc.tf new file mode 100644 index 000000000..0e5c1bc0c --- /dev/null +++ b/5-appinfra/modules/htc-infra/vpc-sc.tf @@ -0,0 +1,122 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_service_account" "cloudbuild_actor" { + project = var.infra_project + account_id = "cloudbuild-actor" + display_name = "Cloudbuild custom service account" +} + +resource "google_service_account" "cloudrun_actor" { + project = var.infra_project + account_id = "cloudrun-actor" + display_name = "Cloud Run custom service account" +} + +data "google_project" "project" { + project_id = var.infra_project +} + +resource "google_access_context_manager_access_level_condition" "access-level-conditions" { + count = var.access_level_name != null ? 1 : 0 + access_level = var.access_level_name + members = concat( + [ + google_service_account.cloudbuild_actor.member, + google_service_account.cloudrun_actor.member + ], + [ + "serviceAccount:service-${data.google_project.project.number}@serverless-robot-prod.iam.gserviceaccount.com", + "serviceAccount:lro-asset-collector@system.gserviceaccount.com", + ]) +} + +resource "google_access_context_manager_service_perimeter_ingress_policy" "ingress_policy" { + count = var.service_perimeter_mode == "ENFORCE" ? 1 : 0 + perimeter = var.service_perimeter_name + title = "Ingress from 848655640797 to Multiple API's" + ingress_from { + identity_type = "ANY_IDENTITY" + sources { + resource = "projects/848655640797" + } + } + ingress_to { + resources = ["*"] + operations { + service_name = "storage.googleapis.com" + method_selectors { + method = "*" + } + } + operations { + service_name = "artifactregistry.googleapis.com" + method_selectors { + method = "*" + } + } + } + lifecycle { + create_before_destroy = true + } + +} + +resource "google_access_context_manager_service_perimeter_ingress_policy" "bq_linked_dataset" { + count = var.service_perimeter_mode == "ENFORCE" ? 1 : 0 + perimeter = var.service_perimeter_name + title = "Allow BigQuery Linked Dataset Creation" + ingress_from { + identity_type = "ANY_IDENTITY" + sources { + resource = "projects/21525134919" + } + } + ingress_to { + resources = ["*"] + operations { + service_name = "logging.googleapis.com" + method_selectors { + method = "*" + } + } + operations { + service_name = "monitoring.googleapis.com" + method_selectors { + method = "*" + } + } + operations { + service_name = "serviceusage.googleapis.com" + method_selectors { + method = "*" + } + } + + } + lifecycle { + create_before_destroy = true + } + +} + +resource "null_resource" "depends_on_vpc_sc_rules" { + depends_on = [ + google_access_context_manager_access_level_condition.access-level-conditions, + google_access_context_manager_service_perimeter_ingress_policy.ingress_policy, + google_access_context_manager_service_perimeter_ingress_policy.bq_linked_dataset + ] +} diff --git a/examples/htc/4-appfactory/terraform.tfvars b/examples/htc/4-appfactory/terraform.tfvars new file mode 100644 index 000000000..4ac1e6e2d --- /dev/null +++ b/examples/htc/4-appfactory/terraform.tfvars @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +applications = { + "htc" = { + "htc" = { + create_infra_project = true + create_admin_project = true + } + } +} + +cloudbuildv2_repository_config = { + repo_type = "GITLABv2" + repositories = { + htc = { + repository_name = "htc-i-r" + repository_url = "https://gitlab.com/user/htc-i-r.git" + } + } + # The Secret ID format is: projects/PROJECT_NUMBER/secrets/SECRET_NAME + gitlab_authorizer_credential_secret_id = "REPLACE_WITH_READ_API_SECRET_ID" + gitlab_read_authorizer_credential_secret_id = "REPLACE_WITH_READ_USER_SECRET_ID" + gitlab_webhook_secret_id = "REPLACE_WITH_WEBHOOK_SECRET_ID" + # If you are using a self-hosted instance, you may change the URL below accordingly + gitlab_enterprise_host_uri = "https://gitlab.com" + # Format is projects/PROJECT/locations/LOCATION/namespaces/NAMESPACE/services/SERVICE + gitlab_enterprise_service_directory = "REPLACE_WITH_SERVICE_DIRECTORY" + # .pem string + gitlab_enterprise_ca_certificate = <", "", ""] +``` + +For example in us-central1: +``` +project_id="" +region="us-central1" +zones=["a", "b", "c", "f"] +``` + +For example in europe-west1: +``` +project_id="" +region="europe-west1" +zones=["b", "c", "d"] +``` + +### Create infrastructure + +Authorize `gcloud` if needed: +```sh +gcloud auth login --quiet --update-adc +``` + +Update the `gcloud` project: + +```bash +gcloud config set project YOUR_PROJECT_ID +``` + +You may need to enable some basic APIs for Terraform to work: +```sh +gcloud services enable iam.googleapis.com cloudresourcemanager.googleapis.com +``` + +Initialize and run terraform: +```sh +terraform init +terraform apply +``` + +NOTE: While running the terraform if the APIs are newly enabled, there may be +timing errors and terraform apply will need to be re-run. + +## Seeing infrastructure & Running Test Workloads + +### See what's from terraform + +Inspect the possible run scripts: +```sh +terraform output +``` + +Key variable outputs: + * local_test_scripts contain a list of shell scripts which you can run for different loadtests. + * get_credentials is the command line to fetch the credentials for kubectl. + * lookerstudio_create_dashboard_url is a link to create a new Lookerstudio Dashboard from a template. + * monitoring_dashboard_url is a custom made monitoring dashboard for loadtest. + +### Running the GUI + +Create a virtual environment: +```sh +python3 -m venv ui/.venv +ui/.venv/bin/python3 -m pip install -r ui/requirements.txt +``` + +Run the Gradio dashboard: +```sh +ui/.venv/bin/python3 ui/main.py generated/config.yaml +``` + +Use port 8080 or preview 8080 in the Cloud Shell (Webpreview). This allows you to load +tests, inspect the jobs from BigQuery (similar to the dashboard), and has some deep +links into the Console. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| access\_level\_name | (VPC-SC) Access Level full name. When providing this variable, additional identities will be added to the access level, these are required to work within an enforced VPC-SC Perimeter. | `string` | `null` | no | +| attestation\_kms\_key | The KMS Key ID to be used by attestor. | `string` | n/a | yes | +| bucket\_kms\_key | KMS Key id to be used to encrypt bucket. | `string` | `null` | no | +| buckets\_force\_destroy | When deleting the bucket for storing CICD artifacts, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no | +| cloudbuildv2\_repository\_config | Configuration for integrating repositories with Cloud Build v2:
- repo\_type: Specifies the type of repository. Supported types are 'GITHUBv2', 'GITLABv2', and 'CSR'.
- repositories: A map of repositories to be created. The key must match the exact name of the repository. Each repository is defined by:
- repository\_name: The name of the repository.
- repository\_url: The URL of the repository.
- github\_secret\_id: (Optional) The personal access token for GitHub authentication.
- github\_app\_id\_secret\_id: (Optional) The application ID for a GitHub App used for authentication.
- gitlab\_read\_authorizer\_credential\_secret\_id: (Optional) The read authorizer credential for GitLab access.
- gitlab\_authorizer\_credential\_secret\_id: (Optional) The authorizer credential for GitLab access.
- gitlab\_webhook\_secret\_id: (Optional) The secret ID for the GitLab WebHook.
- gitlab\_enterprise\_host\_uri: (Optional) The URI of the GitLab Enterprise host this connection is for. If not specified, the default value is https://gitlab.com.
- gitlab\_enterprise\_service\_directory: (Optional) Configuration for using Service Directory to privately connect to a GitLab Enterprise server. This should only be set if the GitLab Enterprise server is hosted on-premises and not reachable by public internet. If this field is left empty, calls to the GitLab Enterprise server will be made over the public internet. Format: projects/{project}/locations/{location}/namespaces/{namespace}/services/{service}.
- gitlab\_enterprise\_ca\_certificate: (Optional) SSL certificate to use for requests to GitLab Enterprise.
Note: When using GITLABv2, specify `gitlab_read_authorizer_credential` and `gitlab_authorizer_credential` and `gitlab_webhook_secret_id`.
Note: When using GITHUBv2, specify `github_pat` and `github_app_id`.
Note: If 'cloudbuildv2\_repository\_config' variable is not configured, CSR (Cloud Source Repositories) will be used by default. | 
object({
    repo_type = string # Supported values are: GITHUBv2, GITLABv2 and CSR
    # repositories to be created
    repositories = map(
      object({
        repository_name = string
        repository_url  = string
      })
    )
    # Credential Config for each repository type
    github_secret_id                            = optional(string)
    github_app_id_secret_id                     = optional(string)
    gitlab_read_authorizer_credential_secret_id = optional(string)
    gitlab_authorizer_credential_secret_id      = optional(string)
    gitlab_webhook_secret_id                    = optional(string)
    gitlab_enterprise_host_uri                  = optional(string)
    gitlab_enterprise_service_directory         = optional(string)
    gitlab_enterprise_ca_certificate            = optional(string)
  })
| n/a | yes | +| environment\_names | A list of environment names. | `list(string)` | n/a | yes | +| envs | Environments | 
map(object({
    network_self_link = string
  }))
| n/a | yes | +| logging\_bucket | Bucket to store logging. | `string` | `null` | no | +| region | CI/CD region | `string` | `"us-central1"` | no | +| remote\_state\_bucket | Backend bucket to load Terraform Remote State Data from previous steps. | `string` | n/a | yes | +| team | Example's team name | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| test\_scripts | Test configuration shell scripts | + + diff --git a/examples/htc/5-appinfra/htc/envs/production/backend.tf b/examples/htc/5-appinfra/htc/envs/production/backend.tf new file mode 100644 index 000000000..0025ff6b4 --- /dev/null +++ b/examples/htc/5-appinfra/htc/envs/production/backend.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + backend "gcs" { + bucket = "UPDATE_INFRA_REPO_STATE" + prefix = "terraform/appinfra/htc/htc/production" + } +} diff --git a/examples/htc/5-appinfra/htc/envs/production/main.tf b/examples/htc/5-appinfra/htc/envs/production/main.tf new file mode 100644 index 000000000..86edaf213 --- /dev/null +++ b/examples/htc/5-appinfra/htc/envs/production/main.tf @@ -0,0 +1,66 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + env = "production" + application_name = "htc" + service_name = "htc" + team_name = "default" + repo_name = "eab-${local.application_name}-${local.service_name}" + repo_branch = "main" +} + +module "app" { + source = "../../modules/cicd-pipeline" + + project_id = local.app_admin_project + region = var.region + env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + + service_name = local.service_name + team_name = local.team_name + repo_name = local.repo_name + repo_branch = local.repo_branch + app_build_trigger_yaml = "cloudbuild.yaml" + + buckets_force_destroy = var.buckets_force_destroy + + cloudbuildv2_repository_config = var.cloudbuildv2_repository_config + workerpool_id = data.terraform_remote_state.bootstrap.outputs.cb_private_workerpool_id + access_level_name = var.access_level_name + logging_bucket = var.logging_bucket + bucket_kms_key = var.bucket_kms_key + + attestation_kms_key = var.attestation_kms_key + attestor_id = contains(var.environment_names, "production") ? data.terraform_remote_state.fleetscope["production"].outputs.attestor_id : data.terraform_remote_state.fleetscope[var.environment_names[0]].outputs.attestor_id + binary_authorization_image = data.terraform_remote_state.bootstrap.outputs.binary_authorization_image + binary_authorization_repository_id = data.terraform_remote_state.bootstrap.outputs.binary_authorization_repository_id +} + +module "htc-infra" { + source = "../../modules/htc-infra" + + service_name = local.service_name + gke_cluster_names = local.gke_cluster_names + infra_project = local.app_infra_project + region = var.region + network_self_link = var.envs[local.env].network_self_link + network_name = local.network_name + team = var.team + admin_project = local.app_admin_project + cluster_project_id = local.cluster_project_id + cluster_project_number = local.cluster_project_number + env = local.env +} diff --git a/examples/htc/5-appinfra/htc/envs/production/outputs.tf b/examples/htc/5-appinfra/htc/envs/production/outputs.tf new file mode 100644 index 000000000..d34d6ea2d --- /dev/null +++ b/examples/htc/5-appinfra/htc/envs/production/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "test_scripts" { + description = "Test configuration shell scripts" + value = module.htc-infra.test_scripts +} diff --git a/examples/htc/5-appinfra/htc/envs/production/remote.tf b/examples/htc/5-appinfra/htc/envs/production/remote.tf new file mode 100644 index 000000000..7c0f60609 --- /dev/null +++ b/examples/htc/5-appinfra/htc/envs/production/remote.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["htc.htc"].app_admin_project_id + app_infra_project = data.terraform_remote_state.appfactory.outputs.app-group["htc.htc"].app_infra_project_ids[local.env] + + gke_cluster_names = data.terraform_remote_state.multitenant[local.env].outputs.cluster_names + network_name = data.terraform_remote_state.multitenant[local.env].outputs.network_names + cluster_project_id = data.terraform_remote_state.multitenant[local.env].outputs.cluster_project_id + cluster_project_number = data.terraform_remote_state.multitenant[local.env].outputs.cluster_project_number +} + +data "terraform_remote_state" "multitenant" { + for_each = toset(var.environment_names) + + backend = "gcs" + + config = { + bucket = var.remote_state_bucket + prefix = "terraform/multi_tenant/${each.value}" + } +} + +data "terraform_remote_state" "bootstrap" { + backend = "gcs" + + config = { + bucket = var.remote_state_bucket + prefix = "terraform/bootstrap" + } +} + +data "terraform_remote_state" "appfactory" { + backend = "gcs" + + config = { + bucket = var.remote_state_bucket + prefix = "terraform/appfactory/shared" + } +} + +data "terraform_remote_state" "fleetscope" { + for_each = toset(var.environment_names) + + backend = "gcs" + + config = { + bucket = var.remote_state_bucket + prefix = "terraform/fleet_scope/${each.value}" + } +} diff --git a/examples/htc/5-appinfra/htc/envs/production/variables.tf b/examples/htc/5-appinfra/htc/envs/production/variables.tf new file mode 100644 index 000000000..fd8227d36 --- /dev/null +++ b/examples/htc/5-appinfra/htc/envs/production/variables.tf @@ -0,0 +1,134 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------- +# Project and Regional Configuration +#----------------------------------------------------- + +variable "region" { + description = "CI/CD region" + type = string + default = "us-central1" +} + +variable "buckets_force_destroy" { + description = "When deleting the bucket for storing CICD artifacts, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects." + type = bool + default = false +} + +variable "remote_state_bucket" { + description = "Backend bucket to load Terraform Remote State Data from previous steps." + type = string +} + +variable "environment_names" { + description = "A list of environment names." + type = list(string) +} + +variable "cloudbuildv2_repository_config" { + description = <<-EOT + Configuration for integrating repositories with Cloud Build v2: + - repo_type: Specifies the type of repository. Supported types are 'GITHUBv2', 'GITLABv2', and 'CSR'. + - repositories: A map of repositories to be created. The key must match the exact name of the repository. Each repository is defined by: + - repository_name: The name of the repository. + - repository_url: The URL of the repository. + - github_secret_id: (Optional) The personal access token for GitHub authentication. + - github_app_id_secret_id: (Optional) The application ID for a GitHub App used for authentication. + - gitlab_read_authorizer_credential_secret_id: (Optional) The read authorizer credential for GitLab access. + - gitlab_authorizer_credential_secret_id: (Optional) The authorizer credential for GitLab access. + - gitlab_webhook_secret_id: (Optional) The secret ID for the GitLab WebHook. + - gitlab_enterprise_host_uri: (Optional) The URI of the GitLab Enterprise host this connection is for. If not specified, the default value is https://gitlab.com. + - gitlab_enterprise_service_directory: (Optional) Configuration for using Service Directory to privately connect to a GitLab Enterprise server. This should only be set if the GitLab Enterprise server is hosted on-premises and not reachable by public internet. If this field is left empty, calls to the GitLab Enterprise server will be made over the public internet. Format: projects/{project}/locations/{location}/namespaces/{namespace}/services/{service}. + - gitlab_enterprise_ca_certificate: (Optional) SSL certificate to use for requests to GitLab Enterprise. + Note: When using GITLABv2, specify `gitlab_read_authorizer_credential` and `gitlab_authorizer_credential` and `gitlab_webhook_secret_id`. + Note: When using GITHUBv2, specify `github_pat` and `github_app_id`. + Note: If 'cloudbuildv2_repository_config' variable is not configured, CSR (Cloud Source Repositories) will be used by default. + EOT + type = object({ + repo_type = string # Supported values are: GITHUBv2, GITLABv2 and CSR + # repositories to be created + repositories = map( + object({ + repository_name = string + repository_url = string + }) + ) + # Credential Config for each repository type + github_secret_id = optional(string) + github_app_id_secret_id = optional(string) + gitlab_read_authorizer_credential_secret_id = optional(string) + gitlab_authorizer_credential_secret_id = optional(string) + gitlab_webhook_secret_id = optional(string) + gitlab_enterprise_host_uri = optional(string) + gitlab_enterprise_service_directory = optional(string) + gitlab_enterprise_ca_certificate = optional(string) + }) + + validation { + condition = ( + var.cloudbuildv2_repository_config.repo_type == "GITHUBv2" ? ( + var.cloudbuildv2_repository_config.github_secret_id != null && + var.cloudbuildv2_repository_config.github_app_id_secret_id != null && + var.cloudbuildv2_repository_config.gitlab_read_authorizer_credential_secret_id == null && + var.cloudbuildv2_repository_config.gitlab_authorizer_credential_secret_id == null && + var.cloudbuildv2_repository_config.gitlab_webhook_secret_id == null + ) : var.cloudbuildv2_repository_config.repo_type == "GITLABv2" ? ( + var.cloudbuildv2_repository_config.github_secret_id == null && + var.cloudbuildv2_repository_config.github_app_id_secret_id == null && + var.cloudbuildv2_repository_config.gitlab_read_authorizer_credential_secret_id != null && + var.cloudbuildv2_repository_config.gitlab_authorizer_credential_secret_id != null && + var.cloudbuildv2_repository_config.gitlab_webhook_secret_id != null + ) : var.cloudbuildv2_repository_config.repo_type == "CSR" ? true : false + ) + error_message = "You must specify a valid repo_type ('GITHUBv2', 'GITLABv2', or 'CSR'). For 'GITHUBv2', all 'github_' prefixed variables must be defined and no 'gitlab_' prefixed variables should be defined. For 'GITLABv2', all 'gitlab_' prefixed variables must be defined and no 'github_' prefixed variables should be defined." + } + +} + +variable "access_level_name" { + description = "(VPC-SC) Access Level full name. When providing this variable, additional identities will be added to the access level, these are required to work within an enforced VPC-SC Perimeter." + type = string + default = null +} + +variable "logging_bucket" { + description = "Bucket to store logging." + type = string + default = null +} + +variable "bucket_kms_key" { + description = "KMS Key id to be used to encrypt bucket." + type = string + default = null +} + +variable "attestation_kms_key" { + type = string + description = "The KMS Key ID to be used by attestor." +} + +variable "team" { + description = "Example's team name" + type = string +} + +variable "envs" { + description = "Environments" + type = map(object({ + network_self_link = string + })) +} diff --git a/examples/htc/5-appinfra/htc/envs/production/versions.tf b/examples/htc/5-appinfra/htc/envs/production/versions.tf new file mode 100644 index 000000000..fc108a017 --- /dev/null +++ b/examples/htc/5-appinfra/htc/envs/production/versions.tf @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.5.3" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.4" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + } + provider_meta "google" { + module_name = "cloud-solutions/fsi-rdp-loadtest-v1.0.0" + } +} diff --git a/examples/htc/5-appinfra/htc/modules b/examples/htc/5-appinfra/htc/modules new file mode 120000 index 000000000..09cfb95eb --- /dev/null +++ b/examples/htc/5-appinfra/htc/modules @@ -0,0 +1 @@ +../../../../5-appinfra/modules \ No newline at end of file diff --git a/examples/htc/5-appinfra/htc/tf-wrapper.sh b/examples/htc/5-appinfra/htc/tf-wrapper.sh new file mode 100755 index 000000000..1d7a650a8 --- /dev/null +++ b/examples/htc/5-appinfra/htc/tf-wrapper.sh @@ -0,0 +1,343 @@ +#!/bin/bash + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +action=$1 +branch=$2 +policy_source=$3 +project_id=$4 +policy_type=$5 # FILESYSTEM | CLOUDSOURCE +runner_env=$6 # GITHUB | CLOUDBUILD | JENKINS | LOCAL +base_dir=$(pwd) +tmp_plan="${base_dir}/tmp_plan" #if you change this, update build triggers + +#==============================================================================# +# Configuration for the search depth of folders in the source code that +# contains the terraform configurations. +#==============================================================================# +max_depth=1 # Must be configured based in your directory design +min_depth=1 # Must be configured based in your directory design + + +#==============================================================================# +# The regex to find folders that contains the Terraform configurations to apply. +# +# When using environments as leaf nodes (default) the regex contains the there +# branches/environments development, nonproduction, and "production" and the +# additional special value "shared" +# +# When using environments as root nodes the regex contains the name of the +# folder that contain the Terraform configuration e.g: business_unit_1 +# and business_unit_2 +#==============================================================================# + +# Environments as leaf nodes in source code case +leaf_regex_plan="^(development|nonproduction|production|shared)$" + +# Environments as root nodes in source code case +# leaf_regex_plan="^(business_unit_1|business_unit_2)$" + +#====================================================================# +# Function used for the criteria for running terraform int/plan/show +# and gcloud beta terraform vet for all the Terraform configurations. +#====================================================================# +do_plan() { + local leaf + leaf="$(basename "$1")" + if [[ "$leaf" =~ $leaf_regex_plan ]] ; then + echo "true" + else + echo "false" + fi +} + +#=========================================================# +# Environments as leaf nodes in source code case (Default) +# Example: +# git-repo +# └── business_unit_1 +# ├── development +# ├── nonproduction +# └── production +# └── business_unit_2 +# ├── development +# ├── nonproduction +# └── production +#=========================================================# + +##### Start of default source organization - comment to use Environments as root nodes ##### +do_action() { + local leaf + leaf="$(basename "$1")" + if [[ "$leaf" == "$branch" ]] || [[ "$leaf" == "shared" && "$branch" == "production" ]]; then + echo "true" + else + echo "false" + fi +} +##### End of default source organization ##### + +#=============================================================# +# Environments as root nodes in source code Case (alternative) +# Example: +# git-repo +# └── development +# ├── business_unit_1 +# └── business_unit_2 +# └── nonproduction +# ├── business_unit_1 +# └── business_unit_2 +# └── production +# ├── business_unit_1 +# └── business_unit_2 +#=============================================================# + +##### Start of alternative source organization - uncomment to use Environments as root nodes ##### + +# leaf_regex_action="^(business_unit_1|business_unit_2)$" # edit this list +# do_action() { +# local env_path="$1" +# local tf_env="${env_path#$base_dir/}" +# local tf_leaf +# local tf_root +# tf_leaf="$(basename "$env_path")" +# tf_root="$(echo "$tf_env" | cut -d/ -f1)" +# if [[ "$tf_leaf" =~ $leaf_regex_action ]] && [[ "$tf_root" == "$branch" ]] ; then +# echo "true" +# else +# if [[ "$tf_leaf" =~ $leaf_regex_action && "$tf_root" == "shared" && "$branch" == "production" ]]; then +# echo "true" +# else +# echo "false" +# fi +# fi +# } + +##### End of alternative source organization ##### + +#====================================================================# +# Function to replace '/' with '-' to convert a path to a file name +#====================================================================# +convert_path() { + echo "$1" | sed -r 's/\//-/g' +} + +## Terraform apply for single environment. +tf_apply() { + local path=$1 + local tf_env="${path#"$base_dir"/}" + local tf_file + tf_file="$(convert_path "$tf_env")" + echo "*************** TERRAFORM APPLY *******************" + echo " At environment: ${tf_env} " + echo "***************************************************" + if [ -d "$path" ]; then + cd "$path" || exit + terraform apply -no-color -input=false -auto-approve "${tmp_plan}/${tf_file}.tfplan" || exit 1 + cd "$base_dir" || exit + else + echo "ERROR: ${path} does not exist" + fi +} + +## terraform init for single environment. +tf_init() { + local path=$1 + local tf_env="${path#"$base_dir"/}" + echo "*************** TERRAFORM INIT *******************" + echo " At environment: ${tf_env} " + echo "**************************************************" + if [ -d "$path" ]; then + cd "$path" || exit + terraform init -no-color || exit 11 + cd "$base_dir" || exit + else + echo "ERROR: ${path} does not exist" + fi +} + +## terraform plan for single environment. +tf_plan() { + local path=$1 + local tf_env="${path#"$base_dir"/}" + local tf_file + tf_file="$(convert_path "$tf_env")" + echo "*************** TERRAFORM PLAN *******************" + echo " At environment: ${tf_env} " + echo "**************************************************" + if [ ! -d "${tmp_plan}" ]; then + mkdir "${tmp_plan}" || exit + fi + if [ -d "$path" ]; then + cd "$path" || exit + terraform plan -no-color -input=false -out "${tmp_plan}/${tf_file}.tfplan" || exit 21 + cd "$base_dir" || exit + else + echo "ERROR: ${path} does not exist" + fi +} + +#============================================================================# +# terraform init/plan/validate for all valid environments matching condition. +#============================================================================# +tf_plan_validate_all() { + local leaf + find "$base_dir" -mindepth 1 -maxdepth 1 -type d \ + -not -path "$base_dir/modules" \ + -not -path "$base_dir/.git" \ + -not -path "$base_dir/.terraform" | while read -r component_path ; do + find "$component_path" -mindepth "$min_depth" -maxdepth "$max_depth" -type d | while read -r env_path ; do + if [[ "$(do_plan "$env_path")" == "true" ]] ; then + tf_init "$env_path" + tf_plan "$env_path" + tf_validate "$env_path" "$policy_source" + else + echo "${env_path#"$base_dir"/} doesn't match $leaf_regex_plan; skipping" + fi + done + done +} + +## terraform show for single environment. +tf_show() { + local path=$1 + local tf_env="${path#"$base_dir"/}" + local tf_file + tf_file="$(convert_path "$tf_env")" + echo "*************** TERRAFORM SHOW *******************" + echo " At environment: ${tf_env} " + echo "**************************************************" + if [ -d "$path" ]; then + cd "$path" || exit + terraform show -no-color "${tmp_plan}/${tf_file}.tfplan" || exit 41 + cd "$base_dir" || exit + else + echo "ERROR: ${path} does not exist" + fi +} + +## terraform validate for single environment. +tf_validate() { + local path=$1 + local policy_file_path=$2 + local tf_env="${path#"$base_dir"/}" + local tf_file + tf_file="${tmp_plan}/$(convert_path "$tf_env")" + echo "*************** TERRAFORM VALIDATE ******************" + echo " At environment: ${tf_env} " + echo " Using policy from: ${policy_file_path} " + echo "*****************************************************" + if [ -z "$policy_file_path" ]; then + echo "no policy repo found! Check the argument provided for policy_source to this script." + echo "https://github.com/GoogleCloudPlatform/policy-library/blob/main/docs/user_guide.md#how-to-set-up-constraints-with-policy-library" + else + if [ -d "$path" ]; then + cd "$path" || exit + # In GitHub actions environment 'terraform' is not the terraform binary but a wrapper around it + # that prints the command 'terraform show' itself in the redirection to the json file, making + # the json file to have an invalid format. 'terraform-bin' is the actual terraform binary. + if [[ "$runner_env" == "GITHUB" ]]; then + terraform-bin show -no-color -json "${tf_file}.tfplan" > "${tf_file}.json" || exit 32 + terraform-bin show -no-color "${tf_file}.tfplan" > "${tf_file}.txt" || exit 36 + else + terraform show -no-color -json "${tf_file}.tfplan" > "${tf_file}.json" || exit 32 + terraform show -no-color "${tf_file}.tfplan" > "${tf_file}.txt" || exit 36 + fi + if [[ "$policy_type" == "CLOUDSOURCE" ]]; then + # Check if $policy_file_path is empty so we clone the policies repo only once + if [ -z "$(ls -A "${policy_file_path}" 2> /dev/null)" ]; then + gcloud source repos clone gcp-policies "${policy_file_path}" --project="${project_id}" || exit 34 + pushd . + cd "${policy_file_path}" + # Commented command below works only on Git 2.22.0+ + # current_branch=$(git branch --show-current) + # As Cloud Build is based on step 4-projects docker image having + # git version 2.20.1 installed the command below keeps compatibility + current_branch=$(git symbolic-ref --short HEAD) + echo "current gcp-policies branch $current_branch" + if [[ "$current_branch" != "main" ]]; then + git checkout main || exit 35 + fi + popd + fi + fi + gcloud beta terraform vet "${tf_file}.json" --policy-library="${policy_file_path}" --project="${project_id}" || exit 33 + cd "$base_dir" || exit + else + echo "ERROR: ${path} does not exist" + fi + fi +} + +#=================================================================# +# Runs single action for each instance of env in folder hierarchy. +#=================================================================# +single_action_runner() { + local leaf + # filter folders that does not contain Terraform configurations + find "$base_dir" -mindepth 1 -maxdepth 1 -type d \ + -not -path "$base_dir/modules" \ + -not -path "$base_dir/.git" \ + -not -path "$base_dir/.terraform" | while read -r component_path ; do + # sort -r is added to ensure shared is first if it exists. + find "$component_path" -mindepth "$min_depth" -maxdepth "$max_depth" -type d | sort -r | while read -r env_path ; do + if [[ "$(do_action "$env_path")" == "true" ]]; then + case "$action" in + apply ) + tf_apply "$env_path" + ;; + + init ) + tf_init "$env_path" + ;; + + plan ) + tf_plan "$env_path" + ;; + + show ) + tf_show "$env_path" + ;; + + validate ) + tf_validate "$env_path" "$policy_source" + ;; + * ) + echo "unknown option: ${action}" + ;; + esac + else + echo "${env_path#"$base_dir"/} doesn't match ${branch}; skipping" + fi + done + done +} + +case "$action" in + init|plan|apply|show|validate ) + single_action_runner + ;; + + plan_validate_all ) + tf_plan_validate_all + ;; + + * ) + echo "unknown option: ${1}" + exit 99 + ;; +esac diff --git a/examples/htc/6-appsource/cloudbuild.yaml b/examples/htc/6-appsource/cloudbuild.yaml new file mode 100644 index 000000000..82de4edb3 --- /dev/null +++ b/examples/htc/6-appsource/cloudbuild.yaml @@ -0,0 +1,119 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - id: 'get-agent-image-digest' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: + - '-c' + - | + IMAGE_URI="$_CONTAINER_REGISTRY/agent:latest" + AGENT_DIGEST=$(gcloud artifacts docker images describe "$$IMAGE_URI" \ + --format='get(image_summary.digest)') + echo "Found digest for agent: $$AGENT_DIGEST" + # Store the digest in a file for subsequent steps + echo "export AGENT_DIGEST=$$AGENT_DIGEST" >> /workspace/digests.env + # We write to /workspace/digests.env, which is shared across steps + + - id: 'get-loadtest-image-digest' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: + - '-c' + - | + IMAGE_URI="$_CONTAINER_REGISTRY/loadtest:latest" + LOADTEST_DIGEST=$(gcloud artifacts docker images describe "$$IMAGE_URI" \ + --format='get(image_summary.digest)') + echo "Found digest for loadtest: $$LOADTEST_DIGEST" + # Append the digest to the same file + echo "export LOADTEST_DIGEST=$$LOADTEST_DIGEST" >> /workspace/digests.env + + - id: 'get-american-option-image-digest' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: + - '-c' + - | + IMAGE_URI="$_CONTAINER_REGISTRY/american_option:latest" # Corrected from agent:latest + AMERICANOPTION_DIGEST=$(gcloud artifacts docker images describe "$$IMAGE_URI" \ + --format='get(image_summary.digest)') + echo "Found digest for american-option: $$AMERICANOPTION_DIGEST" + # Append the digest to the same file + echo "export AMERICANOPTION_DIGEST=$$AMERICANOPTION_DIGEST" >> /workspace/digests.env + + - id: 'log-image-paths' + name: 'ubuntu' + entrypoint: 'bash' + args: + - '-c' + - | + # Source the file to load the variables into this step's environment + source /workspace/digests.env + echo "Attesting agent image: $_CONTAINER_REGISTRY/agent@$$AGENT_DIGEST" + echo "Attesting loadtest image: $_CONTAINER_REGISTRY/loadtest@$$LOADTEST_DIGEST" + echo "Attesting american-option image: $_CONTAINER_REGISTRY/american_option@$$AMERICANOPTION_DIGEST" + echo "Attestor ID: $_ATTESTOR_ID" + echo "KMS Key Version: $_KMS_KEY_VERSION" + + - id: 'create-agent-attestation' + name: '$_BINARY_AUTH_IMAGE' + entrypoint: 'bash' + args: + - '-c' + - | + # Source the file to load the variables into this step's environment + source /workspace/digests.env + /work/create_binauthz_attestation.sh \ + '--artifact-url' \ + "$_CONTAINER_REGISTRY/agent@$$AGENT_DIGEST" \ + '--attestor' \ + "$_ATTESTOR_ID" \ + '--keyversion' \ + "$_KMS_KEY_VERSION" + + - id: 'create-loadtest-attestation' + name: '$_BINARY_AUTH_IMAGE' + entrypoint: 'bash' + args: + - '-c' + - | + source /workspace/digests.env + /work/create_binauthz_attestation.sh \ + '--artifact-url' \ + "$_CONTAINER_REGISTRY/loadtest@$$LOADTEST_DIGEST" \ + '--attestor' \ + "$_ATTESTOR_ID" \ + '--keyversion' \ + "$_KMS_KEY_VERSION" + + - id: 'create-american-option-attestation' + name: '$_BINARY_AUTH_IMAGE' + entrypoint: 'bash' + args: + - '-c' + - | + source /workspace/digests.env + /work/create_binauthz_attestation.sh \ + '--artifact-url' \ + "$_CONTAINER_REGISTRY/american_option@$$AMERICANOPTION_DIGEST" \ + '--attestor' \ + "$_ATTESTOR_ID" \ + '--keyversion' \ + "$_KMS_KEY_VERSION" + +options: + logging: CLOUD_LOGGING_ONLY + pool: + name: '${_PRIVATE_POOL}' diff --git a/examples/htc/README.md b/examples/htc/README.md new file mode 100644 index 000000000..5614c032e --- /dev/null +++ b/examples/htc/README.md @@ -0,0 +1,264 @@ +# High Throughput Compute Example + +This example shows how to deploy the HTC example using the infrastructure created using [Enterprise Application blueprint](https://cloud.google.com/architecture/enterprise-application-blueprint). + +## Overview + +This is an example of running a loadtest library on Google Cloud infrastructure. + +It will run using GKE horizontal pod autoscaler orchestrated using Pub/Sub. For details on the loadtest library, see +its [README.md](https://github.com/GoogleCloudPlatform/risk-and-research-blueprints/blob/main/examples/risk/loadtest/src/README.md). The same techniques can be used to run any kind of +library that is exposing gRPC. + +Cloud Logging, Pub/Sub, Cloud Monitoring, BigQuery, and Looker Studio will all be used +for monitoring the infrastructure as it scales. + +## Pre-Requisites + +This example requires: + +1. 1-bootstrap phase executed successfully. +1. 2-multitenant phase executed successfully. +1. 3-fleetscope phase executed successfully. + +## Usage + +Please note that some steps in this documentation are specific to the selected Git provider. These steps are clearly marked at the beginning of each instruction. For example, if a step applies only to GitHub users, it will be labeled with "(GitHub only)." + +### Deploying with Google Cloud Build + +The steps below assume that you are checked out on the same level as `terraform-google-enterprise-application` and `terraform-example-foundation` directories. + +```txt +. +├── terraform-example-foundation +├── terraform-google-enterprise-application +└── . +``` + +#### Add HTC namespaces at Fleetscope repository + +1. Navigate to Fleetscope repository and add the HTC namespaces at `terraform.tfvars` if they were not created: + + ```hcl + namespace_ids = { + "htc" = "your-htc-group@yourdomain.com" # Note: Update with your team's Google Group + } + ``` + +1. Commit and push changes. Because the plan branch is not a named environment branch, pushing your plan branch triggers terraform plan but not terraform apply. Review the plan output in your Cloud Build project. + + ```bash + git checkout plan + git add . + git commit -m 'Adds HTC namespaces.' + git push --set-upstream origin plan + ``` + +1. Merge changes to production. Because this is a named environment branch, pushing to this branch triggers both terraform plan and terraform apply. Review the apply output in your Cloud Build project + + ```bash + git checkout production + git merge plan + git push origin production + ``` + +1. Move out of Fleetscope folder: + + ```bash + cd ../ + ``` + +#### Add HTC envs at App Factory + +1. Copy the `examples/htc/4-appfactory` folder content to the repo: + + ```bash + mkdir ./eab-applicationfactory/envs/ + cp -R ./terraform-google-enterprise-application/examples/htc/4-appfactory/envs/* ./eab-applicationfactory/envs/ + cp -R ./terraform-google-enterprise-application/4-appfactory/modules/* ./eab-applicationfactory/modules/ + ``` + +1. Use `terraform output` to get the state bucket value from 1-bootstrap output and replace the placeholder in `terraform.tfvars`. + + ```bash + cd eab-applicationfactory + terraform -chdir="../terraform-google-enterprise-application/1-bootstrap/" init + export remote_state_bucket=$(terraform -chdir="../terraform-google-enterprise-application/1-bootstrap/" output -raw state_bucket) + + echo "remote_state_bucket = ${remote_state_bucket}" + + sed -i'' -e "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" ./terraform.tfvars + + sed -i'' -e "s/UPDATE_ME/${remote_state_bucket}/" ./envs/production/backend.tf + ``` + +1. Navigate to Application Factory repository and checkout plan branch: + + ```bash + git checkout plan + git add . + git commit -m 'Adds HTC code' + git push --set-upstream origin plan + ``` + +1. Merge changes to production. Because this is a named environment branch, pushing to this branch triggers both terraform plan and terraform apply. Review the apply output in your Cloud Build project + + ```bash + git checkout production + git merge plan + git push origin production + ``` + +1. Move out of App Factory folder: + + ```bash + cd ../ + ``` + +#### Add HTC envs at App Infra + +1. Retrieve HTC repositories created on 4-appfactory. + + ```bash + cd eab-applicationfactory/envs/production/ + terraform init + + export htc_project=$(terraform output -json app-group | jq -r '.["htc.htc"]["app_admin_project_id"]') + echo htc_project=$htc_project + export htc_repository=$(terraform output -json app-group | jq -r '.["htc.htc"]["app_infra_repository_name"]') + echo htc_repository=$htc_repository + export htc_statebucket=$(terraform output -json app-group | jq -r '.["htc.htc"]["app_cloudbuild_workspace_state_bucket_name"]' | sed 's/.*\///') + echo htc_statebucket=$htc_statebucket + cd ../../../ + ``` + +1. Use `terraform output` to get the state bucket value from 1-bootstrap output and replace the placeholder in `terraform.tfvars`. + + ```bash + terraform -chdir="./terraform-google-enterprise-application/1-bootstrap/" init + export remote_state_bucket=$(terraform -chdir="./terraform-google-enterprise-application/1-bootstrap/" output -raw state_bucket) + echo "remote_state_bucket = ${remote_state_bucket}" + ``` + +1. (CSR Only) Clone the repositories for each service and initialize: + + ```bash + mkdir htc + cd htc + gcloud source repos clone $htc_repository --project=$htc_project + ``` + +1. (GitHub Only) When using GitHub, clone the repositories for each service and initialize with the following commands. + + ```bash + mkdir htc + cd htc + git clone git@github.com:/$htc_repository.git + ``` + + > NOTE: Make sure to replace `` with your actual GitHub owner or organization name. + + +1. (GitLab Only) When using GitLab, clone the repositories for each service and initialize with the following commands. + + ```bash + mkdir htc + cd htc + git clone git@gitlab.com:/$htc_repository.git + ``` + + > NOTE: Make sure to replace `` with your actual GitLab group or account name. + +1. Copy terraform code for each service repository and replace backend bucket: + + ```bash + rm -rf $htc_repository/modules + + cp -R ../terraform-google-enterprise-application/examples/htc/5-appinfra/htc/* $htc_repository + rm -rf $htc_repository/modules + cp -R ../terraform-google-enterprise-application/5-appinfra/modules $htc_repository + cp ../terraform-example-foundation/build/cloudbuild-tf-* $htc_repository/ + cp ../terraform-example-foundation/build/tf-wrapper.sh $htc_repository/ + chmod 755 $htc_repository/tf-wrapper.sh + cp -RT ../terraform-example-foundation/policy-library/ $htc_repository/policy-library + rm -rf $htc_repository/policy-library/policies/constraints/* + sed -i 's/CLOUDSOURCE/FILESYSTEM/g' $htc_repository/cloudbuild-tf-* + sed -i'' -e "s/UPDATE_INFRA_REPO_STATE/$htc_statebucket/" $htc_repository/*/*/backend.tf + sed -i'' -e "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" $htc_repository/*/*/terraform.tfvars + +##### Commit changes for HTC service + +1. Commit files to htc repository a plan branch: + + ```bash + cd $htc_repository + + git checkout -b plan + git add . + git commit -m 'Initialize htc repo' + git push --set-upstream origin plan + ``` + +1. Merge plan to production branch: + + ```bash + git checkout -b production + git push --set-upstream origin production + ``` + +#### Application Source Code pipeline + +This section describes how to clone the empty application source repository, copy the example code into it, and push it to trigger the CI/CD pipeline. + +1. Retrieve Application Source Repository Details + +First, retrieve the names of the source code repository and its Google Cloud project from the App Factory's Terraform state. These were created during the 4-appfactory stage. + + ```bash + cd eab-applicationfactory/envs/production/ + terraform init + export app_source_project=$(terraform output -json app-group | jq -r '.["htc.htc"]["app_source_project_id"]') + echo app_source_project=$app_source_project + export app_source_repository=$(terraform output -json app-group | jq -r '.["htc.htc"]["app_source_repository_name"]') + echo app_source_repository=$app_source_repository + cd ../../../ + ``` + +1. Clone the empty repository created by the App Factory. (CSR Only) + + ```bash + gcloud source repos clone $app_source_repository --project=$app_source_project + ``` + +1. Clone the empty repository created by the App Factory. (CSR Only) + + ```bash + git clone git@github.com:/$app_source_repository.git + ``` + +1. Clone the empty repository created by the App Factory. (Gitlab Only) + + ```bash + git clone git@gitlab.com:/$app_source_repository.git + ``` + +1. Copy and Commit the Example Source Code + +Define the path to the example source code, copy it into your new repository, and commit the files. + + ```bash + export APP_SOURCE_DIR_PATH=$(readlink -f ./terraform-google-enterprise-application/examples/htc/6-appsource) + cd $app_source_repository + cp -r $APP_SOURCE_DIR_PATH/* ./ + git add . + git commit -m "Add HTC application source code" + ``` + +1. Push to main to Trigger the Pipeline + +Push the code to the main branch. This will trigger the Cloud Build pipeline to build and deploy the HTC application. + + ```bash + git push --set-upstream origin main + ``` From 7e8376d53bd762512cc158d5d4bedc99e43de0ab Mon Sep 17 00:00:00 2001 From: marcelorobj Date: Thu, 4 Dec 2025 14:57:57 -0300 Subject: [PATCH 3/7] fix: addressing pr comments --- 5-appinfra/modules/htc-infra/agent/README.md | 179 --- .../modules/config_apply/k8s/agent_job.templ | 3 +- .../k8s/parallelstore/agent_job.templ | 2 +- .../k8s/parallelstore/controller_job.templ | 1 + .../k8s/parallelstore/hpa.yaml.templ | 2 + .../config_apply/k8s/parallelstore/job.templ | 1 + .../k8s/parallelstore/volume.yaml.templ | 1 + .../k8s/parallelstore/volume_claim.yaml.templ | 1 + .../config_apply/k8s/test_config.sh.templ | 2 +- .../agent/modules/config_apply/main.tf | 6 +- .../agent/modules/config_apply/variables.tf | 40 +- .../dashboards/risk-platform-overview.json | 1255 ----------------- .../htc-infra/agent/modules/gke/main.tf | 12 - .../htc-infra/agent/modules/gke/outputs.tf | 8 +- .../htc-infra/agent/modules/gke/variables.tf | 6 - .../htc-infra/agent/src/protoio/io_test.go | 2 +- .../large-files-test-sidecar/fio-config.yaml | 1 - 5-appinfra/modules/htc-infra/main.tf | 20 - 18 files changed, 34 insertions(+), 1508 deletions(-) delete mode 100644 5-appinfra/modules/htc-infra/agent/README.md delete mode 100644 5-appinfra/modules/htc-infra/agent/modules/gke/dashboards/risk-platform-overview.json diff --git a/5-appinfra/modules/htc-infra/agent/README.md b/5-appinfra/modules/htc-infra/agent/README.md deleted file mode 100644 index 9646632e4..000000000 --- a/5-appinfra/modules/htc-infra/agent/README.md +++ /dev/null @@ -1,179 +0,0 @@ - -# Agent for HTC - -## Overview - -The Agent for HTC is a tool for deploying [Unary GRPC](https://grpc.io/docs/what-is-grpc/core-concepts/#unary-rpc) services. This an RPC service -that takes in a [protobuf message](https://protobuf.dev/overview/) and returns a -protobuf message. - -This can use the [loadtest](../loadtest) or [american-option](../american-option) as -workloads and will be used in this example. Your own workload can be used as well. - -## Agent Modes - -The Agent is intended to be deployed as a sidecar to the gRPC service. It can -be deployed in [Cloud Run](https://cloud.google.com/run) or [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine). Cloud Run -provides scale-to-zero, rapid scaling, and a fully managed service. Google -Kubernetes Engine provides immense flexibility, scalability, and enterprise control. - -The following examples are focused in Cloud Run but some can run on Google Kubernetes -Engine (or locally). - -### BigQuery - -![BigQuery Pattern](docs/bigquery_pattern.png "BigQuery Pattern") - -The agent will be deployed alongside the workload in the same container on Cloud Run. It will -receive HTTP requests from [BigQuery Remote Functions](https://cloud.google.com/bigquery/docs/remote-functions), converting -JSON requests into protobuf, dispatching it to the gRPC service, and returning the -results back to BigQuery as JSON. - -This enables flexible data anslysis from BigQuery (or notebook leveraging BigQuery), -accessing arbitrary gRPC-based services. - -### Pub/Sub Push - -![Pub/Sub Push Pattern](docs/pubsub_push_pattern.png "Pub/Sub Push Pattern") - -The agent will be deployed alongside the workload in the same container on Cloud Run. It will -receive HTTP requests from a [Pub/Sub Push Subscription](https://cloud.google.com/pubsub/docs/push). It will -receive JSON requests, convert into protobuf, dispatching it to the gRPC service, -and publish the protobuf result as JSON into a topic. - -### Pub/Sub - -![Pub/Sub Pattern](docs/pubsub_pattern.png "Pub/Sub Pattern") - -The agent will be deployed alongside the workload in the same container on Cloud Run, -the same container on GKE, or on the same machine. It will -receive HTTP requests from a [Pub/Sub Subscription](https://cloud.google.com/pubsub/docs/overview). It will -receive JSON requests from the subscription, convert into protobuf, dispatch it to the gRPC service, -and publish the protobuf result as JSON into a topic. - -### File IO - -![File Pattern](docs/file_pattern.png "File Pattern") - -The agent will read from a JSONL file, dispatch each line into gRPC, and write the -results into a JSONL output file. - -This is more intended for testing purposes. - -## Test Modes - -The Agent also includes two modes for testing: gRPC (direct) and Pub/Sub. - -For testing it can be used for latency testing (on-going tests, measuring throughput -and latency) or for batch (send all data, wait for all results). - -TO BE ADDED. - -## Testing Locally - -NOTE: It is assumed that loadtest tasks have been generated as `../loadtest/tasks.jsonl`, -and loadtest is running on port 2002. See [loadtest README.md](../loadtest/README.md) -for instructions on getting it running. - -### Build the container - -```sh -docker build -t agent . -``` - -### Running File IO - -The following command: - * Runs the container on the host network (so can connect to the gRPC service) - * Mounts the local loadtest folder (so it can read the task.jsonl) - * Runs the "agent file" subcommand with the input and output files. - * Configures the gRPC endpoint, service, and method for targeting the gRPC service. - -```sh -docker run \ - --network host \ - -v $PWD/../loadtest:/data \ - agent \ - agent file /data/tasks.jsonl /data/tasks_output.jsonl \ - --endpoint http://localhost:2002/main.LoadTestService/RunLibrary -``` - -### Running BigQuery - -Start the agent in BigQuery RDF mode like with File IO. This will open a port -on 8080 (on the host) for HTTP requests. - -```sh -docker run \ - --network host \ - agent \ - --logJSON \ - agent \ - rdf \ - --endpoint http://localhost:2002/main.LoadTestService/RunLibrary -``` - -Create an input.json file with the following tasks: - -```sh -cat > input.json < ./generated/k8s_configs/${each.key} kubectl apply -f - < bool 60)", - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "STACKED_BAR", - "legendTemplate": "", - "targetAxis": "Y2", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "prometheusQuery": "sum(increase(kubernetes_io:container_restart_count{monitored_resource=\"k8s_container\",pod_name=~\"gke-hpa.+\"}[${__interval}]))", - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "STACKED_BAR", - "legendTemplate": "", - "targetAxis": "Y2", - "dimensions": [], - "measures": [], - "breakdowns": [] - } - ], - "thresholds": [], - "y2Axis": { - "label": "Count", - "scale": "LINEAR" - }, - "chartOptions": { - "mode": "COLOR", - "showLegend": false, - "displayHorizontal": false - } - }, - "id": "" - } - }, - { - "yPos": 48, - "width": 48, - "height": 48, - "widget": { - "title": "GKE", - "collapsibleGroup": { - "collapsed": true - }, - "id": "" - } - }, - { - "xPos": 24, - "yPos": 48, - "width": 24, - "height": 16, - "widget": { - "title": "Number of instances per machine type", - "timeSeriesTable": { - "dataSets": [ - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_COUNT", - "groupByFields": [ - "metadata.system_labels.\"machine_type\"" - ] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "tableTemplate": "", - "minAlignmentPeriod": "60s" - } - ], - "metricVisualization": "NUMBER", - "columnSettings": [ - { - "column": "system_labels.machine_type", - "visible": true, - "displayName": "Machine Type" - }, - { - "column": "project_id", - "visible": false - }, - { - "column": "value", - "visible": true - } - ], - "opsAnalyticsSettings": { - "maxRows": "0", - "showFilterBar": false, - "pageSize": "0" - }, - "displayColumnType": false - }, - "id": "" - } - }, - { - "yPos": 64, - "width": 24, - "height": 16, - "widget": { - "title": "CPU - total usage / request / limit", - "xyChart": { - "dataSets": [ - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/cpu/core_usage_time\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_RATE", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y1", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/cpu/limit_cores\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y1", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/cpu/request_cores\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y1", - "dimensions": [], - "measures": [], - "breakdowns": [] - } - ], - "thresholds": [], - "yAxis": { - "label": "", - "scale": "LINEAR" - }, - "chartOptions": { - "mode": "COLOR", - "showLegend": false, - "displayHorizontal": false - } - }, - "id": "" - } - }, - { - "xPos": 24, - "yPos": 64, - "width": 24, - "height": 16, - "widget": { - "title": "Memory - total usage / request / limit", - "xyChart": { - "dataSets": [ - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/memory/used_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y2", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/memory/limit_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y2", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/memory/request_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y2", - "dimensions": [], - "measures": [], - "breakdowns": [] - } - ], - "thresholds": [], - "y2Axis": { - "label": "", - "scale": "LINEAR" - }, - "chartOptions": { - "mode": "COLOR", - "showLegend": false, - "displayHorizontal": false - } - }, - "id": "" - } - }, - { - "yPos": 24, - "width": 48, - "height": 24, - "widget": { - "title": "Worker", - "collapsibleGroup": { - "collapsed": true - }, - "id": "" - } - }, - { - "yPos": 96, - "width": 48, - "height": 32, - "widget": { - "title": "Pub / Sub", - "collapsibleGroup": { - "collapsed": true - }, - "id": "" - } - }, - { - "xPos": 24, - "yPos": 80, - "width": 24, - "height": 16, - "widget": { - "title": "Autoscaler events", - "xyChart": { - "dataSets": [ - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"logging.googleapis.com/log_entry_count\" resource.type=\"k8s_cluster\" metric.label.\"log\"=\"container.googleapis.com/cluster-autoscaler-visibility\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_RATE", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "STACKED_BAR", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y1", - "dimensions": [], - "measures": [], - "breakdowns": [] - } - ], - "thresholds": [], - "yAxis": { - "label": "", - "scale": "LINEAR" - }, - "chartOptions": { - "mode": "COLOR", - "showLegend": false, - "displayHorizontal": false - } - }, - "id": "" - } - }, - { - "yPos": 80, - "width": 24, - "height": 16, - "widget": { - "title": "Ephemeral Storage - total usage / request / limit", - "xyChart": { - "dataSets": [ - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/ephemeral_storage/used_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y2", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/ephemeral_storage/limit_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y2", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"kubernetes.io/container/ephemeral_storage/request_bytes\" resource.type=\"k8s_container\" resource.label.\"pod_name\"=monitoring.regex.full_match(\"gke-hpa.+\")", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "", - "minAlignmentPeriod": "60s", - "targetAxis": "Y2", - "dimensions": [], - "measures": [], - "breakdowns": [] - } - ], - "thresholds": [], - "y2Axis": { - "label": "", - "scale": "LINEAR" - }, - "chartOptions": { - "mode": "COLOR", - "showLegend": false, - "displayHorizontal": false - } - }, - "id": "" - } - }, - { - "yPos": 24, - "width": 24, - "height": 24, - "widget": { - "title": "Worker replica stats", - "xyChart": { - "dataSets": [ - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/kube_horizontalpodautoscaler_spec_min_replicas/gauge\" resource.type=\"prometheus_target\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - }, - "pickTimeSeriesFilter": { - "rankingMethod": "METHOD_MEAN", - "numTimeSeries": 30, - "direction": "TOP" - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "Min Replicas", - "minAlignmentPeriod": "60s", - "targetAxis": "Y1", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/kube_horizontalpodautoscaler_spec_max_replicas/gauge\" resource.type=\"prometheus_target\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - }, - "pickTimeSeriesFilter": { - "rankingMethod": "METHOD_MEAN", - "numTimeSeries": 30, - "direction": "TOP" - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "Max Replicas", - "minAlignmentPeriod": "60s", - "targetAxis": "Y1", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/kube_horizontalpodautoscaler_status_desired_replicas/gauge\" resource.type=\"prometheus_target\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - }, - "pickTimeSeriesFilter": { - "rankingMethod": "METHOD_MEAN", - "numTimeSeries": 30, - "direction": "TOP" - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "Desired Replicas", - "minAlignmentPeriod": "60s", - "targetAxis": "Y1", - "dimensions": [], - "measures": [], - "breakdowns": [] - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/kube_horizontalpodautoscaler_status_current_replicas/gauge\" resource.type=\"prometheus_target\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [] - }, - "pickTimeSeriesFilter": { - "rankingMethod": "METHOD_MEAN", - "numTimeSeries": 30, - "direction": "TOP" - } - }, - "unitOverride": "", - "outputFullDuration": false - }, - "plotType": "LINE", - "legendTemplate": "Current Replicas", - "minAlignmentPeriod": "60s", - "targetAxis": "Y1", - "dimensions": [], - "measures": [], - "breakdowns": [] - } - ], - "thresholds": [], - "yAxis": { - "label": "", - "scale": "LINEAR" - }, - "chartOptions": { - "mode": "COLOR", - "showLegend": false, - "displayHorizontal": false - } - }, - "id": "" - } - } - ] - }, - "dashboardFilters": [], - "labels": {} - } diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf index 98a4874aa..daaeaecc7 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf @@ -67,18 +67,6 @@ resource "google_pubsub_subscription" "subscription" { } } -# Dashboard - -resource "google_monitoring_dashboard" "risk-platform-overview" { - project = data.google_project.environment.project_id - dashboard_json = file("${path.module}/${var.dashboard}") - - lifecycle { - ignore_changes = [ - dashboard_json - ] - } -} # Regional Resources # GCS bucket per region diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf index f2fd2e11d..e32a5a3aa 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf @@ -22,12 +22,6 @@ output "topics" { ] } -# Dashboard for Platform Overview -output "monitoring_dashboard_url" { - description = "Cloud Monitoring dashboard" - value = "https://console.cloud.google.com/monitoring/dashboards/builder/${regex("projects/[0-9]+/dashboards/(.*)$", google_monitoring_dashboard.risk-platform-overview.id)[0]};project=${var.project_id}" -} - # Test scripts (shell scripts) output "test_scripts_list" { description = "Test configuration shell scripts as a list" @@ -40,7 +34,7 @@ output "test_scripts_list" { output "first_test_script" { description = "First test script" - value = module.config_apply[0].test_scripts + value = length(var.gke_cluster_names) > 0 ? module.config_apply[0].test_scripts : "" } # Cluster diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf index d6acc1469..0aae7d8fc 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf @@ -69,12 +69,6 @@ variable "workload_args" { description = "Workload image for Cloud Run templates" } -variable "dashboard" { - type = string - default = "dashboards/risk-platform-overview.json" - description = "dashboard JSON configuration. Default is 'dashboards/risk-platform-overview.json'." -} - # # Optional functionality # (Review suggested) diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go b/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go index c52e9529b..b65c8c752 100644 --- a/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go @@ -81,7 +81,7 @@ func TestProtoDefinition(t *testing.T) { //ReadProto(context.Background(), desc, "testInput") // tmpdir := t.TempDir() - tmpdir := "/tmp" + tmpdir := t.TempDir() tmpFile := path.Join(tmpdir, "testOutput") // Produce ten in a row! diff --git a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/fio-config.yaml b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/fio-config.yaml index 34be28fa3..85762d26c 100644 --- a/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/fio-config.yaml +++ b/5-appinfra/modules/htc-infra/kubernetes/storage/fio-testing/k8s/parallelstore/overlays/large-files-test-sidecar/fio-config.yaml @@ -30,7 +30,6 @@ data: invalidate=1 size=100G - Large files [parallel_reads] name=library_parallel_reads_${POD_NAME} filename_format=testfile_* diff --git a/5-appinfra/modules/htc-infra/main.tf b/5-appinfra/modules/htc-infra/main.tf index faac25ddf..a8b8cdec6 100644 --- a/5-appinfra/modules/htc-infra/main.tf +++ b/5-appinfra/modules/htc-infra/main.tf @@ -31,26 +31,6 @@ locals { config.name => config } - ui_config_file = yamlencode({ - "project_id" : var.infra_project, - "region" : var.region, - "pubsub_summary_table" : "${google_bigquery_table.messages_summary.project}.${google_bigquery_table.messages_summary.dataset_id}.${google_bigquery_table.messages_summary.table_id}", - "urls" : { - "dashboard" = module.gke.monitoring_dashboard_url - "cluster" = module.gke.cluster_urls - }, - "tasks" : concat( - length(module.gke) == 0 ? [] : [ - for config in local.test_configs : { - "name" = "GKE ${config.name}", - # "script" = module.gke.first_test_script[config.name], - "script" = module.gke.test_scripts_list[0], - "parallel" = config.parallel, - "description" = config.description, - } - ], - ), - }) parallelstore_instances = var.storage_type == "PARALLELSTORE" ? { for region, instance in module.parallelstore : region => { name = instance.name_short From ce8a094bc6b9c75a39194d2aaf0085186dfd87f5 Mon Sep 17 00:00:00 2001 From: marcelorobj Date: Fri, 5 Dec 2025 17:27:13 -0300 Subject: [PATCH 4/7] fix: addressing lint errors --- 5-appinfra/modules/htc-infra/README.md | 1 - 5-appinfra/modules/htc-infra/agent/modules/gke/main.tf | 4 ---- 5-appinfra/modules/htc-infra/outputs.tf | 5 ----- 3 files changed, 10 deletions(-) diff --git a/5-appinfra/modules/htc-infra/README.md b/5-appinfra/modules/htc-infra/README.md index 7cbaaab67..94f4da8a2 100644 --- a/5-appinfra/modules/htc-infra/README.md +++ b/5-appinfra/modules/htc-infra/README.md @@ -142,6 +142,5 @@ links into the Console. | Name | Description | |------|-------------| | test\_scripts | Test configuration shell scripts | -| ui\_config | Yaml configuration for UI deployment | diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf index daaeaecc7..40ee33aab 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/main.tf @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -data "google_project" "environment" { - project_id = var.project_id -} - locals { enable_jobs = (var.gke_job_request != "" && var.gke_job_response != "") ? 1 : 0 enable_hpa = (var.gke_hpa_request != "" && var.gke_job_response != "") ? 1 : 0 diff --git a/5-appinfra/modules/htc-infra/outputs.tf b/5-appinfra/modules/htc-infra/outputs.tf index 024ed00b1..0dfbaed62 100644 --- a/5-appinfra/modules/htc-infra/outputs.tf +++ b/5-appinfra/modules/htc-infra/outputs.tf @@ -12,11 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -output "ui_config" { - description = "Yaml configuration for UI deployment" - value = local.ui_config_file -} - output "test_scripts" { description = "Test configuration shell scripts" value = module.gke.test_scripts_list From 78404f89920751dfd1242451a1a469ec3df5737f Mon Sep 17 00:00:00 2001 From: marcelorobj Date: Thu, 11 Dec 2025 14:03:46 -0300 Subject: [PATCH 5/7] fix: adding tls to custom metrics --- .../cicd-pipeline/project-iam-bindings.tf | 2 +- .../k8s/adapter_new_resource_model.yaml.templ | 9 ++- .../agent/modules/config_apply/main.tf | 2 +- .../agent/modules/config_apply/privateca.tf | 68 +++++++++++++++++++ .../agent/modules/config_apply/variables.tf | 20 ------ .../htc-infra/agent/modules/gke/outputs.tf | 5 -- 5-appinfra/modules/htc-infra/apis.tf | 9 ++- examples/htc/README.md | 2 +- 8 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/privateca.tf diff --git a/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf b/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf index 9ea8632cc..86de3df78 100644 --- a/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf +++ b/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf @@ -156,7 +156,7 @@ module "cb-gke-project-iam-bindings" { for_each = local.gke_projects project_id = each.value - project_roles = ["roles/container.admin", "roles/container.developer", "roles/gkehub.viewer", "roles/gkehub.gatewayEditor", "roles/cloudbuild.workerPoolUser", "roles/containeranalysis.notes.attacher"] + project_roles = ["roles/privateca.admin", "roles/container.admin", "roles/container.developer", "roles/gkehub.viewer", "roles/gkehub.gatewayEditor", "roles/cloudbuild.workerPoolUser", "roles/containeranalysis.notes.attacher"] prefix = "serviceAccount" service_account_address = google_service_account.cloud_build.email } diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ index 0052389ba..bba4c7bf9 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ @@ -144,7 +144,8 @@ kind: APIService metadata: name: v1beta1.custom.metrics.k8s.io spec: - insecureSkipTLSVerify: true + insecureSkipTLSVerify: false + caBundle: ${ca_bundle} group: custom.metrics.k8s.io groupPriorityMinimum: 100 versionPriority: 100 @@ -158,7 +159,8 @@ kind: APIService metadata: name: v1beta2.custom.metrics.k8s.io spec: - insecureSkipTLSVerify: true + insecureSkipTLSVerify: false + caBundle: ${ca_bundle} group: custom.metrics.k8s.io groupPriorityMinimum: 100 versionPriority: 200 @@ -172,7 +174,8 @@ kind: APIService metadata: name: v1beta1.external.metrics.k8s.io spec: - insecureSkipTLSVerify: true + insecureSkipTLSVerify: false + caBundle: ${ca_bundle} group: external.metrics.k8s.io groupPriorityMinimum: 100 versionPriority: 100 diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf b/5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf index 097558b6e..bdf27940a 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf @@ -126,6 +126,7 @@ resource "null_resource" "cluster_init" { "adapter_new_resource_model_yaml" = templatefile( "${path.module}/k8s/adapter_new_resource_model.yaml.templ", { namespace = var.namespace + ca_bundle = local.ca_bundle_base64 }), "volume_claim_yaml" = templatefile( "${path.module}/k8s/volume_claim.yaml.templ", { @@ -137,7 +138,6 @@ resource "null_resource" "cluster_init" { triggers = { template = each.value cluster_change = local.cluster_config - test = true } provisioner "local-exec" { diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/privateca.tf b/5-appinfra/modules/htc-infra/agent/modules/config_apply/privateca.tf new file mode 100644 index 000000000..e3793de77 --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/privateca.tf @@ -0,0 +1,68 @@ + +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + ca_bundle_base64 = base64encode(google_privateca_certificate_authority.custom_metrics_ca.pem_ca_certificates[0]) +} + +resource "google_privateca_ca_pool" "custom_metrics_ca_pool" { + project = var.cluster_project_id + location = var.region + name = "custom-metrics-ca-pool" + tier = "DEVOPS" + + publishing_options { + publish_ca_cert = true + publish_crl = false + } +} + +resource "google_privateca_certificate_authority" "custom_metrics_ca" { + project = var.cluster_project_id + location = var.region + pool = google_privateca_ca_pool.custom_metrics_ca_pool.name + certificate_authority_id = "custom-metrics-stackdriver-adapter-ca" + lifetime = "31536000s" # 1 year + deletion_protection = false + + config { + subject_config { + subject { + organization = "HTC Infra" + common_name = "custom-metrics-ca" + } + } + x509_config { + ca_options { + is_ca = true + } + key_usage { + base_key_usage { + cert_sign = true + crl_sign = true + } + extended_key_usage { + server_auth = false + client_auth = false + } + } + } + } + key_spec { + algorithm = "RSA_PKCS1_4096_SHA256" + } +} diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/variables.tf b/5-appinfra/modules/htc-infra/agent/modules/config_apply/variables.tf index a54b0a242..11b6af43e 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/config_apply/variables.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/variables.tf @@ -138,50 +138,30 @@ variable "parallelstore_access_points" { type = string default = null description = "Access points for the parallel store. Must be provided if parallelstore_enabled is true." - validation { - condition = var.parallelstore_enabled ? var.parallelstore_access_points != null : true - error_message = "parallelstore_access_points must be set when parallelstore_enabled is true." - } } variable "parallelstore_vpc_name" { type = string default = null description = "Name of the VPC associated with the parallel store. Required when parallelstore_enabled is true." - validation { - condition = var.parallelstore_enabled ? var.parallelstore_vpc_name != null : true - error_message = "parallelstore_vpc_name must be set when parallelstore_enabled is true." - } } variable "parallelstore_location" { type = string default = null description = "Location of the parallel store. Mandatory if parallelstore_enabled is true." - validation { - condition = var.parallelstore_enabled ? var.parallelstore_location != null : true - error_message = "parallelstore_location must be set when parallelstore_enabled is true." - } } variable "parallelstore_instance_name" { type = string default = null description = "Name of the parallel store instance. Must be specified when parallelstore_enabled is true." - validation { - condition = var.parallelstore_enabled ? var.parallelstore_instance_name != null : true - error_message = "parallelstore_instance_name must be set when parallelstore_enabled is true." - } } variable "parallelstore_capacity_gib" { type = number default = null description = "Capacity of the parallel store in GiB. Required if parallelstore_enabled is true." - validation { - condition = var.parallelstore_enabled ? var.parallelstore_capacity_gib != null : true - error_message = "parallelstore_capacity_gib must be set when parallelstore_enabled is true." - } } variable "namespace" { diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf index e32a5a3aa..de0fa5162 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/outputs.tf @@ -32,11 +32,6 @@ output "test_scripts_list" { ]) } -output "first_test_script" { - description = "First test script" - value = length(var.gke_cluster_names) > 0 ? module.config_apply[0].test_scripts : "" -} - # Cluster output "cluster_urls" { description = "Cluster urls" diff --git a/5-appinfra/modules/htc-infra/apis.tf b/5-appinfra/modules/htc-infra/apis.tf index 766e8568e..6373fab6f 100644 --- a/5-appinfra/modules/htc-infra/apis.tf +++ b/5-appinfra/modules/htc-infra/apis.tf @@ -23,7 +23,8 @@ module "enabled_google_apis" { "bigquery.googleapis.com", "parallelstore.googleapis.com", "container.googleapis.com", - "logging.googleapis.com" + "logging.googleapis.com", + "privateca.googleapis.com" ] } @@ -34,7 +35,8 @@ resource "google_project_iam_member" "team_roles_infra_project" { "roles/pubsub.publisher", "roles/pubsub.viewer", "roles/pubsub.subscriber", - "roles/monitoring.viewer" + "roles/monitoring.viewer", + "roles/privateca.admin" ]) project = var.infra_project @@ -49,7 +51,8 @@ resource "google_project_iam_member" "team_roles_cluster_project" { "roles/pubsub.publisher", "roles/pubsub.viewer", "roles/pubsub.subscriber", - "roles/monitoring.viewer" + "roles/monitoring.viewer", + "roles/privateca.admin" ]) project = var.cluster_project_id diff --git a/examples/htc/README.md b/examples/htc/README.md index 5614c032e..060b8176f 100644 --- a/examples/htc/README.md +++ b/examples/htc/README.md @@ -121,7 +121,7 @@ The steps below assume that you are checked out on the same level as `terraform- 1. Retrieve HTC repositories created on 4-appfactory. ```bash - cd eab-applicationfactory/envs/production/ + cd eab-applicationfactory/envs/shared/ terraform init export htc_project=$(terraform output -json app-group | jq -r '.["htc.htc"]["app_admin_project_id"]') From a4390fd31759cb487b5b96e6beaa86a9d81f99ff Mon Sep 17 00:00:00 2001 From: marcelorobj Date: Fri, 16 Jan 2026 17:23:54 -0300 Subject: [PATCH 6/7] fix: addressing pr comments --- 2-multitenant/envs/development/README.md | 1 + 2-multitenant/envs/development/main.tf | 1 + 2-multitenant/envs/development/variables.tf | 6 + 2-multitenant/envs/nonproduction/README.md | 1 + 2-multitenant/envs/nonproduction/main.tf | 1 + 2-multitenant/envs/nonproduction/variables.tf | 6 + 2-multitenant/envs/production/README.md | 1 + 2-multitenant/envs/production/main.tf | 1 + 2-multitenant/envs/production/variables.tf | 6 + 2-multitenant/modules/env_baseline/README.md | 1 + 2-multitenant/modules/env_baseline/main.tf | 2 + .../modules/env_baseline/variables.tf | 6 + .../modules/app-group-baseline/main.tf | 3 +- .../cicd-pipeline/project-iam-bindings.tf | 2 +- 5-appinfra/modules/htc-infra/README.md | 146 - .../k8s/adapter_new_resource_model.yaml.templ | 212 - .../modules/config_apply/k8s/hpa.yaml.templ | 59 +- .../k8s/keda/keda-2.18.3-core.yaml.templ | 11177 ++++++++++++++++ .../agent/modules/config_apply/main.tf | 203 +- .../agent/modules/config_apply/privateca.tf | 68 - .../agent/modules/config_apply/variables.tf | 20 +- .../agent/modules/config_apply/versions.tf | 11 +- .../htc-infra/agent/modules/gke/main.tf | 28 +- .../htc-infra/agent/modules/gke/variables.tf | 10 + .../htc-infra/agent/modules/gke/versions.tf | 3 - .../modules/htc-infra/agent/src/gcp/google.go | 3 - .../htc-infra/agent/src/gcp/storage.go | 4 +- 5-appinfra/modules/htc-infra/agent/src/go.mod | 2 +- .../modules/htc-infra/agent/src/main.go | 9 +- .../htc-infra/agent/src/protoio/backend.go | 5 +- .../modules/htc-infra/agent/src/protoio/io.go | 3 +- .../htc-infra/agent/src/protoio/io_test.go | 2 +- .../agent/src/protoio/test_service.proto | 2 +- .../htc-infra/agent/src/serve/bigquery_rdf.go | 5 +- .../modules/htc-infra/agent/src/serve/file.go | 5 +- .../htc-infra/agent/src/serve/pubsub.go | 7 +- .../htc-infra/agent/src/serve/pubsub_push.go | 7 +- .../htc-infra/agent/src/serve/serve.go | 7 +- .../htc-infra/agent/src/test/generate.go | 5 +- .../modules/htc-infra/agent/src/test/grpc.go | 7 +- .../htc-infra/agent/src/test/pubsub.go | 7 +- .../htc-infra/agent/src/test/source.go | 7 +- 5-appinfra/modules/htc-infra/apis.tf | 4 +- 5-appinfra/modules/htc-infra/loadtest/go.mod | 2 +- .../modules/htc-infra/loadtest/request.proto | 2 +- .../modules/htc-infra/loadtest/response.proto | 2 +- .../modules/htc-infra/loadtest/service.proto | 2 +- 5-appinfra/modules/htc-infra/main.tf | 12 + .../modules/artifact-registry/versions.tf | 3 - .../htc-infra/modules/builder/versions.tf | 3 - .../modules/htc-infra/modules/keda/keda.tf | 102 + .../modules/htc-infra/modules/keda/outputs.tf | 25 + .../htc-infra/modules/keda/variables.tf | 34 + .../htc-infra/modules/keda/versions.tf | 40 + .../htc-infra/modules/kubectl/versions.tf | 3 - .../htc-infra/modules/lustre/versions.tf | 3 - .../modules/parallelstore/versions.tf | 3 - .../htc-infra/modules/project/versions.tf | 3 - .../modules/pubsub-subscriptions/versions.tf | 3 - .../htc-infra/modules/quota/versions.tf | 3 - .../modules/region-analysis/versions.tf | 3 - 5-appinfra/modules/htc-infra/versions.tf | 3 - .../QUICKSTART.md | 4 +- .../agent/src/go.mod | 2 +- .../agent/src/main.go | 9 +- .../agent/src/protoio/backend.go | 5 +- .../agent/src/protoio/io.go | 3 +- .../agent/src/protoio/io_test.go | 2 +- .../agent/src/protoio/test_service.proto | 2 +- .../agent/src/serve/bigquery_rdf.go | 5 +- .../agent/src/serve/file.go | 5 +- .../agent/src/serve/pubsub.go | 7 +- .../agent/src/serve/pubsub_push.go | 7 +- .../agent/src/serve/serve.go | 7 +- .../agent/src/test/generate.go | 5 +- .../agent/src/test/grpc.go | 7 +- .../agent/src/test/pubsub.go | 7 +- .../agent/src/test/source.go | 7 +- .../modules/infrastructure/README.md | 2 +- .../src/go.mod | 2 +- .../src/request.proto | 2 +- .../src/response.proto | 2 +- .../src/service.proto | 2 +- examples/hpc/README.md | 4 +- examples/htc/6-appsource/cloudbuild.yaml | 62 +- examples/htc/README.md | 43 +- 86 files changed, 11818 insertions(+), 702 deletions(-) delete mode 100644 5-appinfra/modules/htc-infra/README.md delete mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ create mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/keda/keda-2.18.3-core.yaml.templ delete mode 100644 5-appinfra/modules/htc-infra/agent/modules/config_apply/privateca.tf create mode 100644 5-appinfra/modules/htc-infra/modules/keda/keda.tf create mode 100644 5-appinfra/modules/htc-infra/modules/keda/outputs.tf create mode 100644 5-appinfra/modules/htc-infra/modules/keda/variables.tf create mode 100644 5-appinfra/modules/htc-infra/modules/keda/versions.tf diff --git a/2-multitenant/envs/development/README.md b/2-multitenant/envs/development/README.md index ba1544e76..1964b697f 100644 --- a/2-multitenant/envs/development/README.md +++ b/2-multitenant/envs/development/README.md @@ -8,6 +8,7 @@ | cb\_private\_workerpool\_project\_id | Private Worker Pool Project ID used for Cloud Build Triggers. | `string` | `""` | no | | cluster\_release\_channel | The release channel for the clusters | `string` | `"REGULAR"` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| enable\_csi\_gcs\_fuse | Enable the GCS Fuse CSI Driver for HTC example | `bool` | `false` | no | | envs | Environments | 
map(object({
    billing_account    = string
    folder_id          = string
    network_project_id = string
    network_self_link  = string
    org_id             = string
    subnets_self_links = list(string)
  }))
| n/a | yes | | service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"ENFORCE"` | no | | service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no | diff --git a/2-multitenant/envs/development/main.tf b/2-multitenant/envs/development/main.tf index e42d63bd5..079167d30 100644 --- a/2-multitenant/envs/development/main.tf +++ b/2-multitenant/envs/development/main.tf @@ -35,4 +35,5 @@ module "env" { deletion_protection = var.deletion_protection cb_private_workerpool_project_id = var.cb_private_workerpool_project_id cluster_release_channel = var.cluster_release_channel + enable_csi_gcs_fuse = var.enable_csi_gcs_fuse } diff --git a/2-multitenant/envs/development/variables.tf b/2-multitenant/envs/development/variables.tf index 8d7f94f63..a30bc9153 100644 --- a/2-multitenant/envs/development/variables.tf +++ b/2-multitenant/envs/development/variables.tf @@ -86,3 +86,9 @@ variable "cluster_release_channel" { type = string default = "REGULAR" } + +variable "enable_csi_gcs_fuse" { + description = "Enable the GCS Fuse CSI Driver for HTC example" + type = bool + default = false +} diff --git a/2-multitenant/envs/nonproduction/README.md b/2-multitenant/envs/nonproduction/README.md index 08ec6eb54..c577bb569 100644 --- a/2-multitenant/envs/nonproduction/README.md +++ b/2-multitenant/envs/nonproduction/README.md @@ -8,6 +8,7 @@ | cb\_private\_workerpool\_project\_id | Private Worker Pool Project ID used for Cloud Build Triggers. | `string` | `""` | no | | cluster\_release\_channel | The release channel for the clusters | `string` | `"REGULAR"` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| enable\_csi\_gcs\_fuse | Enable the GCS Fuse CSI Driver for HTC example | `bool` | `false` | no | | envs | Environments | 
map(object({
    billing_account    = string
    folder_id          = string
    network_project_id = string
    network_self_link  = string
    org_id             = string
    subnets_self_links = list(string)
  }))
| n/a | yes | | service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"ENFORCE"` | no | | service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no | diff --git a/2-multitenant/envs/nonproduction/main.tf b/2-multitenant/envs/nonproduction/main.tf index 43a8067ee..bcb40352e 100644 --- a/2-multitenant/envs/nonproduction/main.tf +++ b/2-multitenant/envs/nonproduction/main.tf @@ -34,4 +34,5 @@ module "env" { deletion_protection = var.deletion_protection cb_private_workerpool_project_id = var.cb_private_workerpool_project_id cluster_release_channel = var.cluster_release_channel + enable_csi_gcs_fuse = var.enable_csi_gcs_fuse } diff --git a/2-multitenant/envs/nonproduction/variables.tf b/2-multitenant/envs/nonproduction/variables.tf index 22c60f19e..1c705e72b 100644 --- a/2-multitenant/envs/nonproduction/variables.tf +++ b/2-multitenant/envs/nonproduction/variables.tf @@ -85,3 +85,9 @@ variable "cluster_release_channel" { type = string default = "REGULAR" } + +variable "enable_csi_gcs_fuse" { + description = "Enable the GCS Fuse CSI Driver for HTC example" + type = bool + default = false +} diff --git a/2-multitenant/envs/production/README.md b/2-multitenant/envs/production/README.md index 08ec6eb54..c577bb569 100644 --- a/2-multitenant/envs/production/README.md +++ b/2-multitenant/envs/production/README.md @@ -8,6 +8,7 @@ | cb\_private\_workerpool\_project\_id | Private Worker Pool Project ID used for Cloud Build Triggers. | `string` | `""` | no | | cluster\_release\_channel | The release channel for the clusters | `string` | `"REGULAR"` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| enable\_csi\_gcs\_fuse | Enable the GCS Fuse CSI Driver for HTC example | `bool` | `false` | no | | envs | Environments | 
map(object({
    billing_account    = string
    folder_id          = string
    network_project_id = string
    network_self_link  = string
    org_id             = string
    subnets_self_links = list(string)
  }))
| n/a | yes | | service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"ENFORCE"` | no | | service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no | diff --git a/2-multitenant/envs/production/main.tf b/2-multitenant/envs/production/main.tf index 574fccac9..30056848e 100644 --- a/2-multitenant/envs/production/main.tf +++ b/2-multitenant/envs/production/main.tf @@ -34,4 +34,5 @@ module "env" { deletion_protection = var.deletion_protection cb_private_workerpool_project_id = var.cb_private_workerpool_project_id cluster_release_channel = var.cluster_release_channel + enable_csi_gcs_fuse = var.enable_csi_gcs_fuse } diff --git a/2-multitenant/envs/production/variables.tf b/2-multitenant/envs/production/variables.tf index 22c60f19e..1c705e72b 100644 --- a/2-multitenant/envs/production/variables.tf +++ b/2-multitenant/envs/production/variables.tf @@ -85,3 +85,9 @@ variable "cluster_release_channel" { type = string default = "REGULAR" } + +variable "enable_csi_gcs_fuse" { + description = "Enable the GCS Fuse CSI Driver for HTC example" + type = bool + default = false +} diff --git a/2-multitenant/modules/env_baseline/README.md b/2-multitenant/modules/env_baseline/README.md index 78391b084..dced53737 100644 --- a/2-multitenant/modules/env_baseline/README.md +++ b/2-multitenant/modules/env_baseline/README.md @@ -25,6 +25,7 @@ The following resources are created: | create\_cluster\_project | Create Cluster Project ID, otherwise the Network Project ID is used | `bool` | `true` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_csi\_gcs\_fuse | Enable the GCS Fuse CSI Driver for HTC example | `bool` | `false` | no | | env | The environment to prepare (ex. development) | `string` | n/a | yes | | folder\_id | Folder ID | `string` | n/a | yes | | master\_ipv4\_cidr\_blocks | List of IP ranges (One range per cluster) in CIDR notation to use for the hosted master network. This range will be used for assigning private IP addresses to the cluster master(s) and the ILB VIP. This range must not overlap with any other ranges in use within the cluster's network, and it must be a /28 subnet. | `list(string)` | 
[
  "10.11.10.0/28",
  "10.11.20.0/28"
]
| no | diff --git a/2-multitenant/modules/env_baseline/main.tf b/2-multitenant/modules/env_baseline/main.tf index 18c4a7d45..afeb43bdc 100644 --- a/2-multitenant/modules/env_baseline/main.tf +++ b/2-multitenant/modules/env_baseline/main.tf @@ -321,6 +321,8 @@ module "gke-standard" { data.google_compute_default_service_account.compute_sa, ] + gcs_fuse_csi_driver = var.enable_csi_gcs_fuse + // Private Cluster Configuration enable_private_nodes = true enable_private_endpoint = true diff --git a/2-multitenant/modules/env_baseline/variables.tf b/2-multitenant/modules/env_baseline/variables.tf index 2ae8be783..9dcb7d48f 100644 --- a/2-multitenant/modules/env_baseline/variables.tf +++ b/2-multitenant/modules/env_baseline/variables.tf @@ -127,3 +127,9 @@ variable "deletion_protection" { description = "Whether or not to allow Terraform to destroy the cluster." default = true } + +variable "enable_csi_gcs_fuse" { + description = "Enable the GCS Fuse CSI Driver for HTC example" + type = bool + default = false +} diff --git a/4-appfactory/modules/app-group-baseline/main.tf b/4-appfactory/modules/app-group-baseline/main.tf index 581a6f9f0..9f63d38e1 100644 --- a/4-appfactory/modules/app-group-baseline/main.tf +++ b/4-appfactory/modules/app-group-baseline/main.tf @@ -39,7 +39,8 @@ locals { "roles/resourcemanager.projectIamAdmin", "roles/gkehub.admin", "roles/modelarmor.admin", //permission to create model armor template - "roles/iam.serviceAccountAdmin" + "roles/iam.serviceAccountAdmin", + "roles/container.admin" ] } } diff --git a/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf b/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf index 86de3df78..b7e70f2a1 100644 --- a/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf +++ b/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf @@ -156,7 +156,7 @@ module "cb-gke-project-iam-bindings" { for_each = local.gke_projects project_id = each.value - project_roles = ["roles/privateca.admin", "roles/container.admin", "roles/container.developer", "roles/gkehub.viewer", "roles/gkehub.gatewayEditor", "roles/cloudbuild.workerPoolUser", "roles/containeranalysis.notes.attacher"] + project_roles = ["roles/privateca.certificateManager", "roles/container.admin", "roles/container.developer", "roles/gkehub.viewer", "roles/gkehub.gatewayEditor", "roles/cloudbuild.workerPoolUser", "roles/containeranalysis.notes.attacher"] prefix = "serviceAccount" service_account_address = google_service_account.cloud_build.email } diff --git a/5-appinfra/modules/htc-infra/README.md b/5-appinfra/modules/htc-infra/README.md deleted file mode 100644 index 94f4da8a2..000000000 --- a/5-appinfra/modules/htc-infra/README.md +++ /dev/null @@ -1,146 +0,0 @@ - -# Load Test for HTC - -## Overview - -This is a general purpose gRPC load test. - -See the details of running the program in its [src/README.md](src/README.md). See below for -deployment on Google Cloud. - -## Deployment with Cloud Shell - -The following link will walk you through a quick start in Cloud Shell: - -[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://shell.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fgooglecloudplatform%2Frisk-and-research-blueprints&cloudshell_git_branch=main&cloudshell_workspace=examples%2Frisk%2Floadtest&cloudshell_tutorial=QUICKSTART.md&show=terminal) - -## Deploy with Terraform - -### Requirements - -You must have have the following installed: -* `gcloud` installed (see [installation](https://cloud.google.com/sdk/docs/install)) -* `kubectl` installed (see [install tools](https://kubernetes.io/docs/tasks/tools/)) -* A bash-based shell (Linux or Mac OS/X) - -Note that Cloud Shell meets the requirements. - -### Configuration - -Create `terraform.tfvars` with the following content: -``` -project_id="" -region="" -zones=["", "", ""] -``` - -For example in us-central1: -``` -project_id="" -region="us-central1" -zones=["a", "b", "c", "f"] -``` - -For example in europe-west1: -``` -project_id="" -region="europe-west1" -zones=["b", "c", "d"] -``` - -### Create infrastructure - -Authorize `gcloud` if needed: -```sh -gcloud auth login --quiet --update-adc -``` - -Update the `gcloud` project: - -```bash -gcloud config set project YOUR_PROJECT_ID -``` - -You may need to enable some basic APIs for Terraform to work: -```sh -gcloud services enable iam.googleapis.com cloudresourcemanager.googleapis.com -``` - -Initialize and run terraform: -```sh -terraform init -terraform apply -``` - -NOTE: While running the terraform if the APIs are newly enabled, there may be -timing errors and terraform apply will need to be re-run. - -## Seeing infrastructure & Running Test Workloads - -### See what's from terraform - -Inspect the possible run scripts: -```sh -terraform output -``` - -Key variable outputs: - * local_test_scripts contain a list of shell scripts which you can run for different loadtests. - * get_credentials is the command line to fetch the credentials for kubectl. - * lookerstudio_create_dashboard_url is a link to create a new Lookerstudio Dashboard from a template. - * monitoring_dashboard_url is a custom made monitoring dashboard for loadtest. - -### Running the GUI - -Create a virtual environment: -```sh -python3 -m venv ui/.venv -ui/.venv/bin/python3 -m pip install -r ui/requirements.txt -``` - -Run the Gradio dashboard: -```sh -ui/.venv/bin/python3 ui/main.py generated/config.yaml -``` - -Use port 8080 or preview 8080 in the Cloud Shell (Webpreview). This allows you to load -tests, inspect the jobs from BigQuery (similar to the dashboard), and has some deep -links into the Console. - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| access\_level\_name | (VPC-SC) Access Level full name. When providing this variable, additional identities will be added to the access level, these are required to work within an enforced VPC-SC Perimeter. | `string` | `null` | no | -| additional\_quota\_enabled | Enable quota requests for additional resources | `bool` | `false` | no | -| admin\_project | The admin project where cloudbuild/cloudrun configurations will be managed. | `string` | n/a | yes | -| cluster\_project\_id | The GCP project ID where the cluster is created. | `string` | n/a | yes | -| cluster\_project\_number | The GCP project ID where the cluster is created. | `string` | n/a | yes | -| enable\_csi\_parallelstore | Enable the Parallelstore CSI Driver | `bool` | `true` | no | -| env | The environment to prepare (ex. development) | `string` | n/a | yes | -| gke\_cluster\_names | GKE Cluster Name to be used in configurations | `list(string)` | n/a | yes | -| hsn\_bucket | Enable hierarchical namespace GCS buckets | `bool` | `false` | no | -| infra\_project | The infrastructure project where resources will be managed. | `string` | n/a | yes | -| network\_name | VPC Network Name | `string` | n/a | yes | -| network\_self\_link | VPC Network self link | `string` | n/a | yes | -| parallelstore\_deployment\_type | Parallelstore Instance deployment type (SCRATCH or PERSISTENT) | `string` | `"SCRATCH"` | no | -| pubsub\_exactly\_once | Enable Pub/Sub exactly once subscriptions | `bool` | `true` | no | -| quota\_contact\_email | Contact email for quota requests | `string` | `""` | no | -| region | The region where the cloud resources will be deployed. | `string` | n/a | yes | -| regions | List of regions where GKE clusters should be created. Used for multi-region deployments. | `list(string)` | 
[
  "us-central1"
]
| no | -| service\_name | service name (e.g. 'transactionhistory') | `string` | n/a | yes | -| service\_perimeter\_mode | (VPC-SC) Service perimeter mode: ENFORCE, DRY\_RUN. | `string` | `"DRY_RUN"` | no | -| service\_perimeter\_name | (VPC-SC) Service perimeter name. The created projects in this step will be assigned to this perimeter. | `string` | `null` | no | -| storage\_capacity\_gib | Capacity in GiB for the selected storage system (Parallelstore or Lustre) | `number` | `null` | no | -| storage\_locations | Map of region to location (zone) for storage instances e.g. {"us-central1" = "us-central1-a"} | `map(string)` | `{}` | no | -| storage\_type | The type of storage system to deploy (PARALLELSTORE, LUSTRE, or null for none) | `string` | `null` | no | -| team | Environment Team, must be the same as the fleet scope team | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| test\_scripts | Test configuration shell scripts | - - diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ deleted file mode 100644 index bba4c7bf9..000000000 --- a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/adapter_new_resource_model.yaml.templ +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - name: custom-metrics ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: custom-metrics:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: custom-metrics-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: custom-metrics-resource-reader - namespace: ${namespace} -rules: -- apiGroups: - - "" - resources: - - pods - - nodes - - nodes/stats - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: custom-metrics-resource-reader -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: custom-metrics-resource-reader -subjects: -- kind: ServiceAccount - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} - labels: - run: custom-metrics-stackdriver-adapter - k8s-app: custom-metrics-stackdriver-adapter -spec: - replicas: 1 - selector: - matchLabels: - run: custom-metrics-stackdriver-adapter - k8s-app: custom-metrics-stackdriver-adapter - template: - metadata: - labels: - run: custom-metrics-stackdriver-adapter - k8s-app: custom-metrics-stackdriver-adapter - kubernetes.io/cluster-service: "true" - spec: - serviceAccountName: custom-metrics-stackdriver-adapter - priorityClassName: higher-priority - containers: - - image: gcr.io/gke-release/custom-metrics-stackdriver-adapter:v0.15.1-gke.0 - imagePullPolicy: Always - name: pod-custom-metrics-stackdriver-adapter - command: - - /adapter - - --use-new-resource-model=true - - --fallback-for-container-metrics=true - resources: - limits: - cpu: 250m - memory: 200Mi - requests: - cpu: 250m - memory: 200Mi ---- -apiVersion: v1 -kind: Service -metadata: - labels: - run: custom-metrics-stackdriver-adapter - k8s-app: custom-metrics-stackdriver-adapter - kubernetes.io/cluster-service: 'true' - kubernetes.io/name: Adapter - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 443 - selector: - run: custom-metrics-stackdriver-adapter - k8s-app: custom-metrics-stackdriver-adapter - type: ClusterIP ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - name: v1beta1.custom.metrics.k8s.io -spec: - insecureSkipTLSVerify: false - caBundle: ${ca_bundle} - group: custom.metrics.k8s.io - groupPriorityMinimum: 100 - versionPriority: 100 - service: - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} - version: v1beta1 ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - name: v1beta2.custom.metrics.k8s.io -spec: - insecureSkipTLSVerify: false - caBundle: ${ca_bundle} - group: custom.metrics.k8s.io - groupPriorityMinimum: 100 - versionPriority: 200 - service: - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} - version: v1beta2 ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - name: v1beta1.external.metrics.k8s.io -spec: - insecureSkipTLSVerify: false - caBundle: ${ca_bundle} - group: external.metrics.k8s.io - groupPriorityMinimum: 100 - versionPriority: 100 - service: - name: custom-metrics-stackdriver-adapter - namespace: ${namespace} - version: v1beta1 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: external-metrics-reader -rules: -- apiGroups: - - "external.metrics.k8s.io" - resources: - - "*" - verbs: - - list - - get - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: external-metrics-reader -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: external-metrics-reader -subjects: -- kind: ServiceAccount - name: horizontal-pod-autoscaler - namespace: kube-system diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ index 72e03c750..65347eb2e 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/hpa.yaml.templ @@ -1,42 +1,37 @@ # Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 +# REFACTORED FOR KEDA -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-gcp-auth-${name} # Unique name per deployment + namespace: ${namespace} +spec: + podIdentity: + provider: gcp + +--- +# This REPLACES the standard HorizontalPodAutoscaler +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject metadata: name: ${name} namespace: ${namespace} spec: - minReplicas: 1 - maxReplicas: 2 - metrics: - - external: - metric: - name: pubsub.googleapis.com|subscription|num_undelivered_messages - selector: - matchLabels: - resource.labels.subscription_id: "${gke_hpa_request_sub}" - resource.type: "pubsub_subscription" - resource.labels.project_id: "${pubsub_project_id}" - target: - type: AverageValue - averageValue: 1 - type: External scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: ${name} + name: ${name} # Targets the Deployment defined below + minReplicaCount: 1 + maxReplicaCount: 2 + # Use num_undelivered_messages directly for scaling + triggers: + - type: gcp-pubsub + metadata: + subscriptionName: "projects/${pubsub_project_id}/subscriptions/${gke_hpa_request_sub}" + value: "1" + authenticationRef: + name: keda-gcp-auth-${name} + --- apiVersion: apps/v1 kind: Deployment diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/keda/keda-2.18.3-core.yaml.templ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/keda/keda-2.18.3-core.yaml.templ new file mode 100644 index 000000000..b7be2088d --- /dev/null +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/k8s/keda/keda-2.18.3-core.yaml.templ @@ -0,0 +1,11177 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: keda + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: cloudeventsources.eventing.keda.sh +spec: + group: eventing.keda.sh + names: + kind: CloudEventSource + listKind: CloudEventSourceList + plural: cloudeventsources + singular: cloudeventsource + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Active")].status + name: Active + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: CloudEventSource defines how a KEDA event will be sent to event + sink + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudEventSourceSpec defines the spec of CloudEventSource + properties: + authenticationRef: + description: |- + AuthenticationRef points to the TriggerAuthentication or ClusterTriggerAuthentication object that + is used to authenticate the scaler with the environment + properties: + kind: + description: Kind of the resource being referred to. Defaults + to TriggerAuthentication. + type: string + name: + type: string + required: + - name + type: object + clusterName: + type: string + destination: + description: Destination defines the various ways to emit events + properties: + azureEventGridTopic: + properties: + endpoint: + type: string + required: + - endpoint + type: object + http: + properties: + uri: + type: string + required: + - uri + type: object + type: object + eventSubscription: + description: EventSubscription defines filters for events + properties: + excludedEventTypes: + items: + enum: + - keda.scaledobject.ready.v1 + - keda.scaledobject.failed.v1 + - keda.scaledobject.removed.v1 + - keda.scaledjob.ready.v1 + - keda.scaledjob.failed.v1 + - keda.scaledjob.removed.v1 + - keda.authentication.triggerauthentication.created.v1 + - keda.authentication.triggerauthentication.updated.v1 + - keda.authentication.triggerauthentication.removed.v1 + - keda.authentication.clustertriggerauthentication.created.v1 + - keda.authentication.clustertriggerauthentication.updated.v1 + - keda.authentication.clustertriggerauthentication.removed.v1 + type: string + type: array + includedEventTypes: + items: + enum: + - keda.scaledobject.ready.v1 + - keda.scaledobject.failed.v1 + - keda.scaledobject.removed.v1 + - keda.scaledjob.ready.v1 + - keda.scaledjob.failed.v1 + - keda.scaledjob.removed.v1 + - keda.authentication.triggerauthentication.created.v1 + - keda.authentication.triggerauthentication.updated.v1 + - keda.authentication.triggerauthentication.removed.v1 + - keda.authentication.clustertriggerauthentication.created.v1 + - keda.authentication.clustertriggerauthentication.updated.v1 + - keda.authentication.clustertriggerauthentication.removed.v1 + type: string + type: array + type: object + required: + - destination + type: object + status: + description: CloudEventSourceStatus defines the observed state of CloudEventSource + properties: + conditions: + description: Conditions an array representation to store multiple + Conditions + items: + description: Condition to store the condition state + properties: + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition + type: string + required: + - status + - type + type: object + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: clustercloudeventsources.eventing.keda.sh +spec: + group: eventing.keda.sh + names: + kind: ClusterCloudEventSource + listKind: ClusterCloudEventSourceList + plural: clustercloudeventsources + singular: clustercloudeventsource + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Active")].status + name: Active + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudEventSourceSpec defines the spec of CloudEventSource + properties: + authenticationRef: + description: |- + AuthenticationRef points to the TriggerAuthentication or ClusterTriggerAuthentication object that + is used to authenticate the scaler with the environment + properties: + kind: + description: Kind of the resource being referred to. Defaults + to TriggerAuthentication. + type: string + name: + type: string + required: + - name + type: object + clusterName: + type: string + destination: + description: Destination defines the various ways to emit events + properties: + azureEventGridTopic: + properties: + endpoint: + type: string + required: + - endpoint + type: object + http: + properties: + uri: + type: string + required: + - uri + type: object + type: object + eventSubscription: + description: EventSubscription defines filters for events + properties: + excludedEventTypes: + items: + enum: + - keda.scaledobject.ready.v1 + - keda.scaledobject.failed.v1 + - keda.scaledobject.removed.v1 + - keda.scaledjob.ready.v1 + - keda.scaledjob.failed.v1 + - keda.scaledjob.removed.v1 + - keda.authentication.triggerauthentication.created.v1 + - keda.authentication.triggerauthentication.updated.v1 + - keda.authentication.triggerauthentication.removed.v1 + - keda.authentication.clustertriggerauthentication.created.v1 + - keda.authentication.clustertriggerauthentication.updated.v1 + - keda.authentication.clustertriggerauthentication.removed.v1 + type: string + type: array + includedEventTypes: + items: + enum: + - keda.scaledobject.ready.v1 + - keda.scaledobject.failed.v1 + - keda.scaledobject.removed.v1 + - keda.scaledjob.ready.v1 + - keda.scaledjob.failed.v1 + - keda.scaledjob.removed.v1 + - keda.authentication.triggerauthentication.created.v1 + - keda.authentication.triggerauthentication.updated.v1 + - keda.authentication.triggerauthentication.removed.v1 + - keda.authentication.clustertriggerauthentication.created.v1 + - keda.authentication.clustertriggerauthentication.updated.v1 + - keda.authentication.clustertriggerauthentication.removed.v1 + type: string + type: array + type: object + required: + - destination + type: object + status: + description: CloudEventSourceStatus defines the observed state of CloudEventSource + properties: + conditions: + description: Conditions an array representation to store multiple + Conditions + items: + description: Condition to store the condition state + properties: + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition + type: string + required: + - status + - type + type: object + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: clustertriggerauthentications.keda.sh +spec: + group: keda.sh + names: + kind: ClusterTriggerAuthentication + listKind: ClusterTriggerAuthenticationList + plural: clustertriggerauthentications + shortNames: + - cta + - clustertriggerauth + singular: clustertriggerauthentication + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.podIdentity.provider + name: PodIdentity + type: string + - jsonPath: .spec.secretTargetRef[*].name + name: Secret + type: string + - jsonPath: .spec.env[*].name + name: Env + type: string + - jsonPath: .spec.hashiCorpVault.address + name: VaultAddress + type: string + - jsonPath: .status.scaledobjects + name: ScaledObjects + priority: 1 + type: string + - jsonPath: .status.scaledjobs + name: ScaledJobs + priority: 1 + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterTriggerAuthentication defines how a trigger can authenticate + globally + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TriggerAuthenticationSpec defines the various ways to authenticate + properties: + awsSecretManager: + description: AwsSecretManager is used to authenticate using AwsSecretManager + properties: + credentials: + properties: + accessKey: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + accessSecretKey: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + accessToken: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + required: + - accessKey + - accessSecretKey + type: object + podIdentity: + description: |- + AuthPodIdentity allows users to select the platform native identity + mechanism + properties: + identityAuthorityHost: + description: Set identityAuthorityHost to override the default + Azure authority host. If this is set, then the IdentityTenantID + must also be set + type: string + identityId: + type: string + identityOwner: + description: IdentityOwner configures which identity has to + be used during auto discovery, keda or the scaled workload. + Mutually exclusive with roleArn + enum: + - keda + - workload + type: string + identityTenantId: + description: Set identityTenantId to override the default + Azure tenant id. If this is set, then the IdentityID must + also be set + type: string + provider: + description: PodIdentityProvider contains the list of providers + enum: + - azure-workload + - gcp + - aws + - aws-eks + - none + type: string + roleArn: + description: RoleArn sets the AWS RoleArn to be used. Mutually + exclusive with IdentityOwner + type: string + required: + - provider + type: object + region: + type: string + secrets: + items: + properties: + name: + type: string + parameter: + type: string + secretKey: + type: string + versionId: + type: string + versionStage: + type: string + required: + - name + - parameter + type: object + type: array + required: + - secrets + type: object + azureKeyVault: + description: AzureKeyVault is used to authenticate using Azure Key + Vault + properties: + cloud: + properties: + activeDirectoryEndpoint: + type: string + keyVaultResourceURL: + type: string + type: + type: string + required: + - type + type: object + credentials: + properties: + clientId: + type: string + clientSecret: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + tenantId: + type: string + required: + - clientId + - clientSecret + - tenantId + type: object + podIdentity: + description: |- + AuthPodIdentity allows users to select the platform native identity + mechanism + properties: + identityAuthorityHost: + description: Set identityAuthorityHost to override the default + Azure authority host. If this is set, then the IdentityTenantID + must also be set + type: string + identityId: + type: string + identityOwner: + description: IdentityOwner configures which identity has to + be used during auto discovery, keda or the scaled workload. + Mutually exclusive with roleArn + enum: + - keda + - workload + type: string + identityTenantId: + description: Set identityTenantId to override the default + Azure tenant id. If this is set, then the IdentityID must + also be set + type: string + provider: + description: PodIdentityProvider contains the list of providers + enum: + - azure-workload + - gcp + - aws + - aws-eks + - none + type: string + roleArn: + description: RoleArn sets the AWS RoleArn to be used. Mutually + exclusive with IdentityOwner + type: string + required: + - provider + type: object + secrets: + items: + properties: + name: + type: string + parameter: + type: string + version: + type: string + required: + - name + - parameter + type: object + type: array + vaultUri: + type: string + required: + - secrets + - vaultUri + type: object + boundServiceAccountToken: + items: + properties: + parameter: + type: string + serviceAccountName: + type: string + required: + - parameter + - serviceAccountName + type: object + type: array + configMapTargetRef: + items: + description: AuthConfigMapTargetRef is used to authenticate using + a reference to a config map + properties: + key: + type: string + name: + type: string + parameter: + type: string + required: + - key + - name + - parameter + type: object + type: array + env: + items: + description: |- + AuthEnvironment is used to authenticate using environment variables + in the destination ScaleTarget spec + properties: + containerName: + type: string + name: + type: string + parameter: + type: string + required: + - name + - parameter + type: object + type: array + gcpSecretManager: + properties: + credentials: + properties: + clientSecret: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + required: + - clientSecret + type: object + podIdentity: + description: |- + AuthPodIdentity allows users to select the platform native identity + mechanism + properties: + identityAuthorityHost: + description: Set identityAuthorityHost to override the default + Azure authority host. If this is set, then the IdentityTenantID + must also be set + type: string + identityId: + type: string + identityOwner: + description: IdentityOwner configures which identity has to + be used during auto discovery, keda or the scaled workload. + Mutually exclusive with roleArn + enum: + - keda + - workload + type: string + identityTenantId: + description: Set identityTenantId to override the default + Azure tenant id. If this is set, then the IdentityID must + also be set + type: string + provider: + description: PodIdentityProvider contains the list of providers + enum: + - azure-workload + - gcp + - aws + - aws-eks + - none + type: string + roleArn: + description: RoleArn sets the AWS RoleArn to be used. Mutually + exclusive with IdentityOwner + type: string + required: + - provider + type: object + secrets: + items: + properties: + id: + type: string + parameter: + type: string + version: + type: string + required: + - id + - parameter + type: object + type: array + required: + - secrets + type: object + hashiCorpVault: + description: HashiCorpVault is used to authenticate using Hashicorp + Vault + properties: + address: + type: string + authentication: + description: VaultAuthentication contains the list of Hashicorp + Vault authentication methods + type: string + credential: + description: Credential defines the Hashicorp Vault credentials + depending on the authentication method + properties: + serviceAccount: + type: string + serviceAccountName: + type: string + token: + type: string + type: object + mount: + type: string + namespace: + type: string + role: + type: string + secrets: + items: + description: VaultSecret defines the mapping between the path + of the secret in Vault to the parameter + properties: + key: + type: string + parameter: + type: string + path: + type: string + pkiData: + properties: + altNames: + type: string + commonName: + type: string + format: + type: string + ipSans: + type: string + otherSans: + type: string + ttl: + type: string + uriSans: + type: string + type: object + type: + description: VaultSecretType defines the type of vault secret + type: string + required: + - key + - parameter + - path + type: object + type: array + required: + - address + - authentication + - secrets + type: object + podIdentity: + description: |- + AuthPodIdentity allows users to select the platform native identity + mechanism + properties: + identityAuthorityHost: + description: Set identityAuthorityHost to override the default + Azure authority host. If this is set, then the IdentityTenantID + must also be set + type: string + identityId: + type: string + identityOwner: + description: IdentityOwner configures which identity has to be + used during auto discovery, keda or the scaled workload. Mutually + exclusive with roleArn + enum: + - keda + - workload + type: string + identityTenantId: + description: Set identityTenantId to override the default Azure + tenant id. If this is set, then the IdentityID must also be + set + type: string + provider: + description: PodIdentityProvider contains the list of providers + enum: + - azure-workload + - gcp + - aws + - aws-eks + - none + type: string + roleArn: + description: RoleArn sets the AWS RoleArn to be used. Mutually + exclusive with IdentityOwner + type: string + required: + - provider + type: object + secretTargetRef: + items: + description: AuthSecretTargetRef is used to authenticate using a + reference to a secret + properties: + key: + type: string + name: + type: string + parameter: + type: string + required: + - key + - name + - parameter + type: object + type: array + type: object + status: + description: TriggerAuthenticationStatus defines the observed state of + TriggerAuthentication + properties: + scaledjobs: + type: string + scaledobjects: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: scaledjobs.keda.sh +spec: + group: keda.sh + names: + kind: ScaledJob + listKind: ScaledJobList + plural: scaledjobs + shortNames: + - sj + singular: scaledjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.minReplicaCount + name: Min + type: integer + - jsonPath: .spec.maxReplicaCount + name: Max + type: integer + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Active")].status + name: Active + type: string + - jsonPath: .status.conditions[?(@.type=="Paused")].status + name: Paused + type: string + - jsonPath: .status.triggersTypes + name: Triggers + type: string + - jsonPath: .status.authenticationsTypes + name: Authentications + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: ScaledJob is the Schema for the scaledjobs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ScaledJobSpec defines the desired state of ScaledJob + properties: + envSourceContainerName: + type: string + failedJobsHistoryLimit: + format: int32 + type: integer + jobTargetRef: + description: JobSpec describes how the job execution will look like. + properties: + activeDeadlineSeconds: + description: |- + Specifies the duration in seconds relative to the startTime that the job + may be continuously active before the system tries to terminate it; value + must be positive integer. If a Job is suspended (at creation or through an + update), this timer will effectively be stopped and reset when the Job is + resumed again. + format: int64 + type: integer + backoffLimit: + description: |- + Specifies the number of retries before marking this job failed. + Defaults to 6 + format: int32 + type: integer + backoffLimitPerIndex: + description: |- + Specifies the limit for the number of retries within an + index before marking this index as failed. When enabled the number of + failures per index is kept in the pod's + batch.kubernetes.io/job-index-failure-count annotation. It can only + be set when Job's completionMode=Indexed, and the Pod's restart + policy is Never. The field is immutable. + format: int32 + type: integer + completionMode: + description: |- + completionMode specifies how Pod completions are tracked. It can be + `NonIndexed` (default) or `Indexed`. + + `NonIndexed` means that the Job is considered complete when there have + been .spec.completions successfully completed Pods. Each Pod completion is + homologous to each other. + + `Indexed` means that the Pods of a + Job get an associated completion index from 0 to (.spec.completions - 1), + available in the annotation batch.kubernetes.io/job-completion-index. + The Job is considered complete when there is one successfully completed Pod + for each index. + When value is `Indexed`, .spec.completions must be specified and + `.spec.parallelism` must be less than or equal to 10^5. + In addition, The Pod name takes the form + `$(job-name)-$(index)-$(random-string)`, + the Pod hostname takes the form `$(job-name)-$(index)`. + + More completion modes can be added in the future. + If the Job controller observes a mode that it doesn't recognize, which + is possible during upgrades due to version skew, the controller + skips updates for the Job. + type: string + completions: + description: |- + Specifies the desired number of successfully finished pods the + job should be run with. Setting to null means that the success of any + pod signals the success of all pods, and allows parallelism to have any positive + value. Setting to 1 means that parallelism is limited to 1 and the success of that + pod signals the success of the job. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + managedBy: + description: |- + ManagedBy field indicates the controller that manages a Job. The k8s Job + controller reconciles jobs which don't have this field at all or the field + value is the reserved string `kubernetes.io/job-controller`, but skips + reconciling Jobs with a custom value for this field. + The value must be a valid domain-prefixed path (e.g. acme.io/foo) - + all characters before the first "/" must be a valid subdomain as defined + by RFC 1123. All characters trailing the first "/" must be valid HTTP Path + characters as defined by RFC 3986. The value cannot exceed 63 characters. + This field is immutable. + + This field is beta-level. The job controller accepts setting the field + when the feature gate JobManagedBy is enabled (enabled by default). + type: string + manualSelector: + description: |- + manualSelector controls generation of pod labels and pod selectors. + Leave `manualSelector` unset unless you are certain what you are doing. + When false or unset, the system pick labels unique to this job + and appends those labels to the pod template. When true, + the user is responsible for picking unique labels and specifying + the selector. Failure to pick a unique label may cause this + and other jobs to not function correctly. However, You may see + `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` + API. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector + type: boolean + maxFailedIndexes: + description: |- + Specifies the maximal number of failed indexes before marking the Job as + failed, when backoffLimitPerIndex is set. Once the number of failed + indexes exceeds this number the entire Job is marked as Failed and its + execution is terminated. When left as null the job continues execution of + all of its indexes and is marked with the `Complete` Job condition. + It can only be specified when backoffLimitPerIndex is set. + It can be null or up to completions. It is required and must be + less than or equal to 10^4 when is completions greater than 10^5. + format: int32 + type: integer + parallelism: + description: |- + Specifies the maximum desired number of pods the job should + run at any given time. The actual number of pods running in steady state will + be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), + i.e. when the work left to do is less than max parallelism. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + podFailurePolicy: + description: |- + Specifies the policy of handling failed pods. In particular, it allows to + specify the set of actions and conditions which need to be + satisfied to take the associated action. + If empty, the default behaviour applies - the counter of failed pods, + represented by the jobs's .status.failed field, is incremented and it is + checked against the backoffLimit. This field cannot be used in combination + with restartPolicy=OnFailure. + properties: + rules: + description: |- + A list of pod failure policy rules. The rules are evaluated in order. + Once a rule matches a Pod failure, the remaining of the rules are ignored. + When no rule matches the Pod failure, the default handling applies - the + counter of pod failures is incremented and it is checked against + the backoffLimit. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. + One of onExitCodes and onPodConditions, but not both, can be used in each rule. + properties: + action: + description: |- + Specifies the action taken on a pod failure when the requirements are satisfied. + Possible values are: + + - FailJob: indicates that the pod's job is marked as Failed and all + running pods are terminated. + - FailIndex: indicates that the pod's index is marked as Failed and will + not be restarted. + - Ignore: indicates that the counter towards the .backoffLimit is not + incremented and a replacement pod is created. + - Count: indicates that the pod is handled in the default way - the + counter towards the .backoffLimit is incremented. + Additional values are considered to be added in the future. Clients should + react to an unknown action by skipping the rule. + type: string + onExitCodes: + description: Represents the requirement on the container + exit codes. + properties: + containerName: + description: |- + Restricts the check for exit codes to the container with the + specified name. When null, the rule applies to all containers. + When specified, it should match one the container or initContainer + names in the pod template. + type: string + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Containers completed with success (exit code 0) are + excluded from the requirement check. Possible values are: + + - In: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is in the set of specified values. + - NotIn: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is not in the set of specified values. + Additional values are considered to be added in the future. Clients should + react to an unknown operator by assuming the requirement is not satisfied. + type: string + values: + description: |- + Specifies the set of values. Each returned container exit code (might be + multiple in case of multiple containers) is checked against this set of + values with respect to the operator. The list of values must be ordered + and must not contain duplicates. Value '0' cannot be used for the In operator. + At least one element is required. At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + - values + type: object + onPodConditions: + description: |- + Represents the requirement on the pod conditions. The requirement is represented + as a list of pod condition patterns. The requirement is satisfied if at + least one pattern matches an actual pod condition. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyOnPodConditionsPattern describes a pattern for matching + an actual pod condition type. + properties: + status: + description: |- + Specifies the required Pod condition status. To match a pod condition + it is required that the specified status equals the pod condition status. + Defaults to True. + type: string + type: + description: |- + Specifies the required Pod condition type. To match a pod condition + it is required that specified type equals the pod condition type. + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-type: atomic + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + podReplacementPolicy: + description: |- + podReplacementPolicy specifies when to create replacement Pods. + Possible values are: + - TerminatingOrFailed means that we recreate pods + when they are terminating (has a metadata.deletionTimestamp) or failed. + - Failed means to wait until a previously created Pod is fully terminated (has phase + Failed or Succeeded) before creating a replacement Pod. + + When using podFailurePolicy, Failed is the the only allowed value. + TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. + This is on by default. + type: string + selector: + description: |- + A label query over pods that should match the pod count. + Normally, the system sets this field for you. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + successPolicy: + description: |- + successPolicy specifies the policy when the Job can be declared as succeeded. + If empty, the default behavior applies - the Job is declared as succeeded + only when the number of succeeded pods equals to the completions. + When the field is specified, it must be immutable and works only for the Indexed Jobs. + Once the Job meets the SuccessPolicy, the lingering pods are terminated. + properties: + rules: + description: |- + rules represents the list of alternative rules for the declaring the Jobs + as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, + the "SucceededCriteriaMet" condition is added, and the lingering pods are removed. + The terminal state for such a Job has the "Complete" condition. + Additionally, these rules are evaluated in order; Once the Job meets one of the rules, + other rules are ignored. At most 20 elements are allowed. + items: + description: |- + SuccessPolicyRule describes rule for declaring a Job as succeeded. + Each rule must have at least one of the "succeededIndexes" or "succeededCount" specified. + properties: + succeededCount: + description: |- + succeededCount specifies the minimal required size of the actual set of the succeeded indexes + for the Job. When succeededCount is used along with succeededIndexes, the check is + constrained only to the set of indexes specified by succeededIndexes. + For example, given that succeededIndexes is "1-4", succeededCount is "3", + and completed indexes are "1", "3", and "5", the Job isn't declared as succeeded + because only "1" and "3" indexes are considered in that rules. + When this field is null, this doesn't default to any value and + is never evaluated at any time. + When specified it needs to be a positive integer. + format: int32 + type: integer + succeededIndexes: + description: |- + succeededIndexes specifies the set of indexes + which need to be contained in the actual set of the succeeded indexes for the Job. + The list of indexes must be within 0 to ".spec.completions-1" and + must not contain duplicates. At least one element is required. + The indexes are represented as intervals separated by commas. + The intervals can be a decimal integer or a pair of decimal integers separated by a hyphen. + The number are listed in represented by the first and last element of the series, + separated by a hyphen. + For example, if the completed indexes are 1, 3, 4, 5 and 7, they are + represented as "1,3-5,7". + When this field is null, this field doesn't default to any value + and is never evaluated at any time. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + suspend: + description: |- + suspend specifies whether the Job controller should create Pods or not. If + a Job is created with suspend set to true, no Pods are created by the Job + controller. If a Job is suspended after creation (i.e. the flag goes from + false to true), the Job controller will delete all active Pods associated + with this Job. Users must design their workload to gracefully handle this. + Suspending a Job will reset the StartTime field of the Job, effectively + resetting the ActiveDeadlineSeconds timer too. Defaults to false. + type: boolean + template: + description: |- + Describes the pod that will be created when executing a job. + The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in + the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether + a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: Optional text to prepend to the + name of each environment variable. Must + be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to + execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to + execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + - protocol + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's + value. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: Optional text to prepend to the + name of each environment variable. Must + be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to + execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to + execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + - protocol + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: Optional text to prepend to the + name of each environment variable. Must + be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to + execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to + execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + - protocol + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + resources: + description: |- + Resources is the total amount of CPU and Memory resources required by all + containers in the pod. It supports specifying Requests and Limits for + "cpu" and "memory" resource names only. ResourceClaims are not supported. + + This field enables fine-grained control over resource allocation for the + entire pod, allowing resource sharing among containers in a pod. + + This is an alpha field and requires enabling the PodLevelResources feature + gate. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod + to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to + be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod + that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name and + Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph tree, + default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that + should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers. + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine and + then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a Flocker + dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for + iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate + is on. + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a + Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the + volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about + the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to + project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of + the ScaleIO Protection Domain for the configured + storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + ttlSecondsAfterFinished: + description: |- + ttlSecondsAfterFinished limits the lifetime of a Job that has finished + execution (either Complete or Failed). If this field is set, + ttlSecondsAfterFinished after the Job finishes, it is eligible to be + automatically deleted. When the Job is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the Job won't be automatically deleted. If this field is set to zero, + the Job becomes eligible to be deleted immediately after it finishes. + format: int32 + type: integer + required: + - template + type: object + maxReplicaCount: + format: int32 + type: integer + minReplicaCount: + format: int32 + type: integer + pollingInterval: + format: int32 + type: integer + rollout: + description: Rollout defines the strategy for job rollouts + properties: + propagationPolicy: + type: string + strategy: + type: string + type: object + rolloutStrategy: + type: string + scalingStrategy: + description: ScalingStrategy defines the strategy of Scaling + properties: + customScalingQueueLengthDeduction: + format: int32 + type: integer + customScalingRunningJobPercentage: + type: string + multipleScalersCalculation: + type: string + pendingPodConditions: + items: + type: string + type: array + strategy: + type: string + type: object + successfulJobsHistoryLimit: + format: int32 + type: integer + triggers: + items: + description: ScaleTriggers reference the scaler that will be used + properties: + authenticationRef: + description: |- + AuthenticationRef points to the TriggerAuthentication or ClusterTriggerAuthentication object that + is used to authenticate the scaler with the environment + properties: + kind: + description: Kind of the resource being referred to. Defaults + to TriggerAuthentication. + type: string + name: + type: string + required: + - name + type: object + metadata: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + useCachedMetrics: + type: boolean + required: + - metadata + - type + type: object + type: array + required: + - jobTargetRef + - triggers + type: object + status: + description: ScaledJobStatus defines the observed state of ScaledJob + properties: + Paused: + type: string + authenticationsTypes: + type: string + conditions: + description: Conditions an array representation to store multiple + Conditions + items: + description: Condition to store the condition state + properties: + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition + type: string + required: + - status + - type + type: object + type: array + lastActiveTime: + format: date-time + type: string + triggersTypes: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: scaledobjects.keda.sh +spec: + group: keda.sh + names: + kind: ScaledObject + listKind: ScaledObjectList + plural: scaledobjects + shortNames: + - so + singular: scaledobject + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.scaleTargetKind + name: ScaleTargetKind + type: string + - jsonPath: .spec.scaleTargetRef.name + name: ScaleTargetName + type: string + - jsonPath: .spec.minReplicaCount + name: Min + type: integer + - jsonPath: .spec.maxReplicaCount + name: Max + type: integer + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Active")].status + name: Active + type: string + - jsonPath: .status.conditions[?(@.type=="Fallback")].status + name: Fallback + type: string + - jsonPath: .status.conditions[?(@.type=="Paused")].status + name: Paused + type: string + - jsonPath: .status.triggersTypes + name: Triggers + type: string + - jsonPath: .status.authenticationsTypes + name: Authentications + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: ScaledObject is a specification for a ScaledObject resource + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ScaledObjectSpec is the spec for a ScaledObject resource + properties: + advanced: + description: AdvancedConfig specifies advance scaling options + properties: + horizontalPodAutoscalerConfig: + description: HorizontalPodAutoscalerConfig specifies horizontal + scale config + properties: + behavior: + description: |- + HorizontalPodAutoscalerBehavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + maximum: 3600 + minimum: 0 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + maximum: 3600 + minimum: 0 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + name: + type: string + type: object + restoreToOriginalReplicaCount: + type: boolean + scalingModifiers: + description: ScalingModifiers describes advanced scaling logic + options like formula + properties: + activationTarget: + type: string + formula: + type: string + metricType: + description: |- + MetricTargetType specifies the type of metric being targeted, and should be either + "Value", "AverageValue", or "Utilization" + enum: + - AverageValue + - Value + type: string + target: + type: string + type: object + type: object + cooldownPeriod: + format: int32 + type: integer + fallback: + description: Fallback is the spec for fallback options + properties: + behavior: + default: static + enum: + - static + - currentReplicas + - currentReplicasIfHigher + - currentReplicasIfLower + type: string + failureThreshold: + format: int32 + type: integer + replicas: + format: int32 + type: integer + required: + - failureThreshold + - replicas + type: object + idleReplicaCount: + format: int32 + type: integer + initialCooldownPeriod: + format: int32 + type: integer + maxReplicaCount: + format: int32 + type: integer + minReplicaCount: + format: int32 + type: integer + pollingInterval: + format: int32 + type: integer + scaleTargetRef: + description: ScaleTarget holds the reference to the scale target Object + properties: + apiVersion: + type: string + envSourceContainerName: + type: string + kind: + type: string + name: + type: string + required: + - name + type: object + triggers: + items: + description: ScaleTriggers reference the scaler that will be used + properties: + authenticationRef: + description: |- + AuthenticationRef points to the TriggerAuthentication or ClusterTriggerAuthentication object that + is used to authenticate the scaler with the environment + properties: + kind: + description: Kind of the resource being referred to. Defaults + to TriggerAuthentication. + type: string + name: + type: string + required: + - name + type: object + metadata: + additionalProperties: + type: string + type: object + metricType: + description: |- + MetricTargetType specifies the type of metric being targeted, and should be either + "Value", "AverageValue", or "Utilization" + type: string + name: + type: string + type: + type: string + useCachedMetrics: + type: boolean + required: + - metadata + - type + type: object + type: array + required: + - scaleTargetRef + - triggers + type: object + status: + description: ScaledObjectStatus is the status for a ScaledObject resource + properties: + authenticationsTypes: + type: string + compositeScalerName: + type: string + conditions: + description: Conditions an array representation to store multiple + Conditions + items: + description: Condition to store the condition state + properties: + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition + type: string + required: + - status + - type + type: object + type: array + externalMetricNames: + items: + type: string + type: array + health: + additionalProperties: + description: HealthStatus is the status for a ScaledObject's health + properties: + numberOfFailures: + format: int32 + type: integer + status: + description: HealthStatusType is an indication of whether the + health status is happy or failing + type: string + type: object + type: object + hpaName: + type: string + lastActiveTime: + format: date-time + type: string + originalReplicaCount: + format: int32 + type: integer + pausedReplicaCount: + format: int32 + type: integer + resourceMetricNames: + items: + type: string + type: array + scaleTargetGVKR: + description: GroupVersionKindResource provides unified structure for + schema.GroupVersionKind and Resource + properties: + group: + type: string + kind: + type: string + resource: + type: string + version: + type: string + required: + - group + - kind + - resource + - version + type: object + scaleTargetKind: + type: string + triggersTypes: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: triggerauthentications.keda.sh +spec: + group: keda.sh + names: + kind: TriggerAuthentication + listKind: TriggerAuthenticationList + plural: triggerauthentications + shortNames: + - ta + - triggerauth + singular: triggerauthentication + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.podIdentity.provider + name: PodIdentity + type: string + - jsonPath: .spec.secretTargetRef[*].name + name: Secret + type: string + - jsonPath: .spec.env[*].name + name: Env + type: string + - jsonPath: .spec.hashiCorpVault.address + name: VaultAddress + type: string + - jsonPath: .status.scaledobjects + name: ScaledObjects + priority: 1 + type: string + - jsonPath: .status.scaledjobs + name: ScaledJobs + priority: 1 + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: TriggerAuthentication defines how a trigger can authenticate + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TriggerAuthenticationSpec defines the various ways to authenticate + properties: + awsSecretManager: + description: AwsSecretManager is used to authenticate using AwsSecretManager + properties: + credentials: + properties: + accessKey: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + accessSecretKey: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + accessToken: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + required: + - accessKey + - accessSecretKey + type: object + podIdentity: + description: |- + AuthPodIdentity allows users to select the platform native identity + mechanism + properties: + identityAuthorityHost: + description: Set identityAuthorityHost to override the default + Azure authority host. If this is set, then the IdentityTenantID + must also be set + type: string + identityId: + type: string + identityOwner: + description: IdentityOwner configures which identity has to + be used during auto discovery, keda or the scaled workload. + Mutually exclusive with roleArn + enum: + - keda + - workload + type: string + identityTenantId: + description: Set identityTenantId to override the default + Azure tenant id. If this is set, then the IdentityID must + also be set + type: string + provider: + description: PodIdentityProvider contains the list of providers + enum: + - azure-workload + - gcp + - aws + - aws-eks + - none + type: string + roleArn: + description: RoleArn sets the AWS RoleArn to be used. Mutually + exclusive with IdentityOwner + type: string + required: + - provider + type: object + region: + type: string + secrets: + items: + properties: + name: + type: string + parameter: + type: string + secretKey: + type: string + versionId: + type: string + versionStage: + type: string + required: + - name + - parameter + type: object + type: array + required: + - secrets + type: object + azureKeyVault: + description: AzureKeyVault is used to authenticate using Azure Key + Vault + properties: + cloud: + properties: + activeDirectoryEndpoint: + type: string + keyVaultResourceURL: + type: string + type: + type: string + required: + - type + type: object + credentials: + properties: + clientId: + type: string + clientSecret: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + tenantId: + type: string + required: + - clientId + - clientSecret + - tenantId + type: object + podIdentity: + description: |- + AuthPodIdentity allows users to select the platform native identity + mechanism + properties: + identityAuthorityHost: + description: Set identityAuthorityHost to override the default + Azure authority host. If this is set, then the IdentityTenantID + must also be set + type: string + identityId: + type: string + identityOwner: + description: IdentityOwner configures which identity has to + be used during auto discovery, keda or the scaled workload. + Mutually exclusive with roleArn + enum: + - keda + - workload + type: string + identityTenantId: + description: Set identityTenantId to override the default + Azure tenant id. If this is set, then the IdentityID must + also be set + type: string + provider: + description: PodIdentityProvider contains the list of providers + enum: + - azure-workload + - gcp + - aws + - aws-eks + - none + type: string + roleArn: + description: RoleArn sets the AWS RoleArn to be used. Mutually + exclusive with IdentityOwner + type: string + required: + - provider + type: object + secrets: + items: + properties: + name: + type: string + parameter: + type: string + version: + type: string + required: + - name + - parameter + type: object + type: array + vaultUri: + type: string + required: + - secrets + - vaultUri + type: object + boundServiceAccountToken: + items: + properties: + parameter: + type: string + serviceAccountName: + type: string + required: + - parameter + - serviceAccountName + type: object + type: array + configMapTargetRef: + items: + description: AuthConfigMapTargetRef is used to authenticate using + a reference to a config map + properties: + key: + type: string + name: + type: string + parameter: + type: string + required: + - key + - name + - parameter + type: object + type: array + env: + items: + description: |- + AuthEnvironment is used to authenticate using environment variables + in the destination ScaleTarget spec + properties: + containerName: + type: string + name: + type: string + parameter: + type: string + required: + - name + - parameter + type: object + type: array + gcpSecretManager: + properties: + credentials: + properties: + clientSecret: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + required: + - clientSecret + type: object + podIdentity: + description: |- + AuthPodIdentity allows users to select the platform native identity + mechanism + properties: + identityAuthorityHost: + description: Set identityAuthorityHost to override the default + Azure authority host. If this is set, then the IdentityTenantID + must also be set + type: string + identityId: + type: string + identityOwner: + description: IdentityOwner configures which identity has to + be used during auto discovery, keda or the scaled workload. + Mutually exclusive with roleArn + enum: + - keda + - workload + type: string + identityTenantId: + description: Set identityTenantId to override the default + Azure tenant id. If this is set, then the IdentityID must + also be set + type: string + provider: + description: PodIdentityProvider contains the list of providers + enum: + - azure-workload + - gcp + - aws + - aws-eks + - none + type: string + roleArn: + description: RoleArn sets the AWS RoleArn to be used. Mutually + exclusive with IdentityOwner + type: string + required: + - provider + type: object + secrets: + items: + properties: + id: + type: string + parameter: + type: string + version: + type: string + required: + - id + - parameter + type: object + type: array + required: + - secrets + type: object + hashiCorpVault: + description: HashiCorpVault is used to authenticate using Hashicorp + Vault + properties: + address: + type: string + authentication: + description: VaultAuthentication contains the list of Hashicorp + Vault authentication methods + type: string + credential: + description: Credential defines the Hashicorp Vault credentials + depending on the authentication method + properties: + serviceAccount: + type: string + serviceAccountName: + type: string + token: + type: string + type: object + mount: + type: string + namespace: + type: string + role: + type: string + secrets: + items: + description: VaultSecret defines the mapping between the path + of the secret in Vault to the parameter + properties: + key: + type: string + parameter: + type: string + path: + type: string + pkiData: + properties: + altNames: + type: string + commonName: + type: string + format: + type: string + ipSans: + type: string + otherSans: + type: string + ttl: + type: string + uriSans: + type: string + type: object + type: + description: VaultSecretType defines the type of vault secret + type: string + required: + - key + - parameter + - path + type: object + type: array + required: + - address + - authentication + - secrets + type: object + podIdentity: + description: |- + AuthPodIdentity allows users to select the platform native identity + mechanism + properties: + identityAuthorityHost: + description: Set identityAuthorityHost to override the default + Azure authority host. If this is set, then the IdentityTenantID + must also be set + type: string + identityId: + type: string + identityOwner: + description: IdentityOwner configures which identity has to be + used during auto discovery, keda or the scaled workload. Mutually + exclusive with roleArn + enum: + - keda + - workload + type: string + identityTenantId: + description: Set identityTenantId to override the default Azure + tenant id. If this is set, then the IdentityID must also be + set + type: string + provider: + description: PodIdentityProvider contains the list of providers + enum: + - azure-workload + - gcp + - aws + - aws-eks + - none + type: string + roleArn: + description: RoleArn sets the AWS RoleArn to be used. Mutually + exclusive with IdentityOwner + type: string + required: + - provider + type: object + secretTargetRef: + items: + description: AuthSecretTargetRef is used to authenticate using a + reference to a secret + properties: + key: + type: string + name: + type: string + parameter: + type: string + required: + - key + - name + - parameter + type: object + type: array + type: object + status: + description: TriggerAuthenticationStatus defines the observed state of + TriggerAuthentication + properties: + scaledjobs: + type: string + scaledobjects: + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: keda-operator + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-operator + namespace: ${namespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: keda-operator + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-operator + namespace: ${namespace} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: keda-external-metrics-reader + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-external-metrics-reader +rules: +- apiGroups: + - external.metrics.k8s.io + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: keda-operator + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-operator +rules: +- apiGroups: + - "" + resources: + - configmaps + - configmaps/status + - external + - pods + - secrets + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - limitranges + - serviceaccounts + verbs: + - list + - watch +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get +- apiGroups: + - '*' + resources: + - '*/scale' + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + - statefulsets + verbs: + - list + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - eventing.keda.sh + resources: + - cloudeventsources + - cloudeventsources/status + - clustercloudeventsources + - clustercloudeventsources/status + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - keda.sh + resources: + - clustertriggerauthentications + - clustertriggerauthentications/status + - scaledjobs + - scaledjobs/finalizers + - scaledjobs/status + - scaledobjects + - scaledobjects/finalizers + - scaledobjects/status + - triggerauthentications + - triggerauthentications/status + verbs: + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: keda-operator + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-operator + namespace: ${namespace} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: keda-operator +subjects: +- kind: ServiceAccount + name: keda-operator + namespace: ${namespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: keda-auth-reader + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: keda-operator + namespace: ${namespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: keda-hpa-controller-external-metrics + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: keda-external-metrics-reader +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: keda-operator + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: keda-operator +subjects: +- kind: ServiceAccount + name: keda-operator + namespace: ${namespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: keda-system-auth-delegator + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: keda-operator + namespace: ${namespace} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: keda-metrics-apiserver + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-metrics-apiserver + namespace: ${namespace} +spec: + ports: + - name: https + port: 443 + targetPort: 6443 + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: keda-metrics-apiserver +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: keda-operator + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-operator + namespace: ${namespace} +spec: + ports: + - name: metricsservice + port: 9666 + targetPort: 9666 + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: keda-operator +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: keda-metrics-apiserver + app.kubernetes.io/name: keda-metrics-apiserver + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-metrics-apiserver + namespace: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app: keda-metrics-apiserver + template: + metadata: + labels: + app: keda-metrics-apiserver + name: keda-metrics-apiserver + spec: + containers: + - args: + - --secure-port=6443 + - --zap-log-level=error + - --client-ca-file=/certs/ca.crt + - --tls-cert-file=/certs/tls.crt + - --tls-private-key-file=/certs/tls.key + - --cert-dir=/certs + command: + - /keda-adapter + env: + - name: WATCH_NAMESPACE + value: "" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KEDA_HTTP_DEFAULT_TIMEOUT + value: "" + image: ${keda_apiserver_image} + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 6443 + scheme: HTTPS + initialDelaySeconds: 5 + name: keda-metrics-apiserver + ports: + - containerPort: 6443 + name: https + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /readyz + port: 6443 + scheme: HTTPS + initialDelaySeconds: 5 + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /tmp + name: temp-vol + - mountPath: /certs + name: certificates + readOnly: true + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + serviceAccountName: keda-operator + volumes: + - emptyDir: {} + name: temp-vol + - name: certificates + secret: + defaultMode: 420 + secretName: kedaorg-certs +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: keda-operator + app.kubernetes.io/component: operator + app.kubernetes.io/name: keda-operator + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: keda-operator + namespace: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app: keda-operator + template: + metadata: + labels: + app: keda-operator + name: keda-operator + name: keda-operator + spec: + containers: + - args: + - --leader-elect + - --zap-log-level=info + - --zap-encoder=console + - --zap-time-encoding=rfc3339 + - --enable-cert-rotation=true + command: + - /keda + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WATCH_NAMESPACE + value: "" + - name: KEDA_HTTP_DEFAULT_TIMEOUT + value: "" + image: ${keda_image} + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 25 + name: keda-operator + ports: + - containerPort: 8080 + name: http + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 20 + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /certs + name: certificates + readOnly: true + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + serviceAccountName: keda-operator + terminationGracePeriodSeconds: 10 + volumes: + - name: certificates + secret: + defaultMode: 420 + optional: true + secretName: kedaorg-certs +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + app.kubernetes.io/name: v1beta1.external.metrics.k8s.io + app.kubernetes.io/part-of: keda-operator + app.kubernetes.io/version: 2.18.3 + name: v1beta1.external.metrics.k8s.io +spec: + group: external.metrics.k8s.io + groupPriorityMinimum: 100 + service: + name: keda-metrics-apiserver + namespace: ${namespace} + version: v1beta1 + versionPriority: 100 diff --git a/5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf b/5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf index bdf27940a..4c6bf6978 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/config_apply/main.tf @@ -19,6 +19,8 @@ resource "random_string" "suffix" { } locals { + get_credentials_cmd = "gcloud container fleet memberships get-credentials ${var.cluster_name} --project ${var.cluster_project_id} --location ${var.region}" + workload_init_args = { for idx, args in var.workload_init_args : "job-${idx}-${substr(sha256(jsonencode(args)), 0, 10)}" => { @@ -34,12 +36,6 @@ locals { template_path = "${path.module}/k8s/parallelstore/${fname}" } } - cluster_config = "${var.cluster_name}-${var.region}-${var.cluster_project_id}" - - kubeconfig_script = "gcloud container fleet memberships get-credentials ${var.cluster_name} --project ${var.cluster_project_id} --location ${var.region}" - - - # Test output test_job_template = { for id, cfg in var.test_configs : id => templatefile( @@ -100,11 +96,8 @@ locals { KUBECONFIG = "/tmp/kubeconfig_${var.cluster_name}-${var.cluster_project_id}.yaml" }) } -} -# Apply configurations to the cluster -resource "null_resource" "cluster_init" { - for_each = merge( + cluster_init_files = merge( { for fname in fileset(".", "${path.module}/k8s/*.yaml") : fname => file(fname) }, { "volume_yaml" = templatefile( "${path.module}/k8s/volume.yaml.templ", { @@ -123,75 +116,121 @@ resource "null_resource" "cluster_init" { gke_hpa_request_sub = var.pubsub_hpa_request gke_hpa_response = var.gke_hpa_response }), - "adapter_new_resource_model_yaml" = templatefile( - "${path.module}/k8s/adapter_new_resource_model.yaml.templ", { - namespace = var.namespace - ca_bundle = local.ca_bundle_base64 - }), "volume_claim_yaml" = templatefile( "${path.module}/k8s/volume_claim.yaml.templ", { namespace = var.namespace }) } ) +} + +resource "local_file" "keda_rendered_manifest" { + content = templatefile("${path.module}/k8s/keda/keda-2.18.3-core.yaml.templ", { + namespace = var.namespace + keda_image = var.keda_image + keda_apiserver_image = var.keda_apiserver_image + }) + filename = "${path.module}/k8s/keda/keda-rendered.yaml" +} +resource "null_resource" "keda_install" { triggers = { - template = each.value - cluster_change = local.cluster_config + manifest_sha = sha256(local_file.keda_rendered_manifest.content) + auth_cmd = local.get_credentials_cmd } provisioner "local-exec" { - when = create - command = <<-EOT - ${local.kubeconfig_script} + command = < ./generated/k8s_configs/${each.key} +resource "null_resource" "cluster_init" { + for_each = local.cluster_init_files + + triggers = { + manifest = each.value + auth_cmd = local.get_credentials_cmd + } + + depends_on = [ + null_resource.keda_install + ] + + provisioner "local-exec" { + command = <<-EOT + ${local.get_credentials_cmd} + kubectl apply --server-side -f - < ${path.module}/../../../generated/k8s_configs/job_init/${each.key}.yaml + ${local.get_credentials_cmd} + kubectl apply --server-side --v=6 -f - < ${path.module}/../../../generated/k8s_configs/parallelstore_init/${each.key} - kubectl apply -f - < ${path.module}/../../../generated/k8s_configs/parallelstore_job_init/${each.key}.yaml - kubectl apply -f - < cluster } - source = "../config_apply" - cluster_project_id = var.cluster_project_id - infra_project_id = var.project_id - region = each.value.location - cluster_name = each.value.name - agent_image = var.agent_image - namespace = var.namespace - # Workload options + for_each = { for idx, cluster in local.clusters : idx => cluster } + source = "../config_apply" + cluster_project_id = var.cluster_project_id + infra_project_id = var.project_id + region = each.value.location + cluster_name = each.value.name + agent_image = var.agent_image + namespace = var.namespace workload_image = var.workload_image workload_args = var.workload_args workload_grpc_endpoint = var.workload_grpc_endpoint workload_init_args = var.workload_init_args test_configs = var.test_configs gcs_bucket = google_storage_bucket.gcs_storage_data[each.value.location].id - # set this to empty string if not enabled - pubsub_hpa_request = local.enable_hpa == 1 ? google_pubsub_subscription.subscription[var.gke_hpa_request].name : "" - pubsub_job_request = local.enable_jobs == 1 ? google_pubsub_subscription.subscription[var.gke_job_request].name : "" + pubsub_hpa_request = local.enable_hpa == 1 ? google_pubsub_subscription.subscription[var.gke_hpa_request].name : "" + pubsub_job_request = local.enable_jobs == 1 ? google_pubsub_subscription.subscription[var.gke_job_request].name : "" - # Parallelstore Config parallelstore_enabled = var.parallelstore_enabled parallelstore_access_points = var.parallelstore_enabled ? join(",", var.parallelstore_instances[each.value.location].access_points) : null parallelstore_vpc_name = var.parallelstore_enabled ? var.vpc_name : null parallelstore_location = var.parallelstore_enabled ? var.parallelstore_instances[each.value.location].location : null parallelstore_instance_name = var.parallelstore_enabled ? var.parallelstore_instances[each.value.location].name : null parallelstore_capacity_gib = var.parallelstore_enabled ? var.parallelstore_instances[each.value.location].capacity_gib : null + + keda_image = var.keda_image + keda_apiserver_image = var.keda_apiserver_image } diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf index 0aae7d8fc..8d8ed5ff1 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/variables.tf @@ -58,6 +58,16 @@ variable "workload_image" { description = "Map of image name to configuration (source)" } +variable "keda_image" { + type = string + description = "Map of image name to configuration (source)" +} + +variable "keda_apiserver_image" { + type = string + description = "Map of image name to configuration (source)" +} + # Sidecar configuration variable "workload_grpc_endpoint" { type = string diff --git a/5-appinfra/modules/htc-infra/agent/modules/gke/versions.tf b/5-appinfra/modules/htc-infra/agent/modules/gke/versions.tf index 028dbac5f..d6198b469 100644 --- a/5-appinfra/modules/htc-infra/agent/modules/gke/versions.tf +++ b/5-appinfra/modules/htc-infra/agent/modules/gke/versions.tf @@ -29,7 +29,4 @@ terraform { version = ">= 3.7.2" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/agent/src/gcp/google.go b/5-appinfra/modules/htc-infra/agent/src/gcp/google.go index 3a739f57a..317291fac 100644 --- a/5-appinfra/modules/htc-infra/agent/src/gcp/google.go +++ b/5-appinfra/modules/htc-infra/agent/src/gcp/google.go @@ -194,9 +194,6 @@ func (cfg *GoogleConfig) PubSubClient(ctxt context.Context) (*pubsub.Client, err } opts = append(opts, option.WithEndpoint(endpoint)) - // Tracking Options - opts = append(opts, option.WithUserAgent("cloud-solutions/fsi-rdp-agent-v1.0.0")) - // Create the client client, err := pubsub.NewClient(ctxt, cfg.ProjectID, opts...) if err != nil { diff --git a/5-appinfra/modules/htc-infra/agent/src/gcp/storage.go b/5-appinfra/modules/htc-infra/agent/src/gcp/storage.go index 73e15f67b..6bd38252d 100644 --- a/5-appinfra/modules/htc-infra/agent/src/gcp/storage.go +++ b/5-appinfra/modules/htc-infra/agent/src/gcp/storage.go @@ -26,12 +26,10 @@ import ( "strings" "cloud.google.com/go/storage" - "google.golang.org/api/option" ) func (cfg *GoogleConfig) NewStorageClient(ctxt context.Context) (*storage.Client, error) { - return storage.NewClient(ctxt, option.WithUserAgent( - "cloud-solutions/fsi-rdp-agent-v1.0.0")) + return storage.NewClient(ctxt) } type gzipReadWrapper struct { diff --git a/5-appinfra/modules/htc-infra/agent/src/go.mod b/5-appinfra/modules/htc-infra/agent/src/go.mod index 3b329820b..a7fe11689 100644 --- a/5-appinfra/modules/htc-infra/agent/src/go.mod +++ b/5-appinfra/modules/htc-infra/agent/src/go.mod @@ -1,4 +1,4 @@ -module github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent +module github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent go 1.24 toolchain go1.24.1 diff --git a/5-appinfra/modules/htc-infra/agent/src/main.go b/5-appinfra/modules/htc-infra/agent/src/main.go index 138a792d2..a2d46b707 100644 --- a/5-appinfra/modules/htc-infra/agent/src/main.go +++ b/5-appinfra/modules/htc-infra/agent/src/main.go @@ -24,10 +24,11 @@ import ( "syscall" "time" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/serve" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/test" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/serve" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/test" + "github.com/spf13/cobra" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/backend.go b/5-appinfra/modules/htc-infra/agent/src/protoio/backend.go index 734d01d5f..22a8a8ca0 100644 --- a/5-appinfra/modules/htc-infra/agent/src/protoio/backend.go +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/backend.go @@ -24,8 +24,9 @@ import ( "log/slog" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/jhump/protoreflect/grpcreflect" "github.com/spf13/cobra" "google.golang.org/grpc" diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/io.go b/5-appinfra/modules/htc-infra/agent/src/protoio/io.go index d0638190d..29d358491 100644 --- a/5-appinfra/modules/htc-infra/agent/src/protoio/io.go +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/io.go @@ -22,7 +22,8 @@ import ( "iter" "log/slog" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go b/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go index b65c8c752..751e54c86 100644 --- a/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/io_test.go @@ -21,7 +21,7 @@ import ( "google.golang.org/protobuf/proto" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" ) // Command to compile the protobuf for tests: diff --git a/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.proto b/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.proto index 36657a536..947a53db8 100644 --- a/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.proto +++ b/5-appinfra/modules/htc-infra/agent/src/protoio/test_service.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package main; -option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio"; +option go_package = "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio"; service TestService { diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/bigquery_rdf.go b/5-appinfra/modules/htc-infra/agent/src/serve/bigquery_rdf.go index fc0c92c2a..15a3bc060 100644 --- a/5-appinfra/modules/htc-infra/agent/src/serve/bigquery_rdf.go +++ b/5-appinfra/modules/htc-infra/agent/src/serve/bigquery_rdf.go @@ -23,8 +23,9 @@ import ( "net/http" "os" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/file.go b/5-appinfra/modules/htc-infra/agent/src/serve/file.go index 1a7b27f0b..be5fbdd70 100644 --- a/5-appinfra/modules/htc-infra/agent/src/serve/file.go +++ b/5-appinfra/modules/htc-infra/agent/src/serve/file.go @@ -15,8 +15,9 @@ package serve import ( - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" "google.golang.org/protobuf/proto" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/pubsub.go b/5-appinfra/modules/htc-infra/agent/src/serve/pubsub.go index 49a5d4c22..5907f4b6a 100644 --- a/5-appinfra/modules/htc-infra/agent/src/serve/pubsub.go +++ b/5-appinfra/modules/htc-infra/agent/src/serve/pubsub.go @@ -20,10 +20,11 @@ import ( "log/slog" "time" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "cloud.google.com/go/pubsub" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" "github.com/spf13/cobra" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/pubsub_push.go b/5-appinfra/modules/htc-infra/agent/src/serve/pubsub_push.go index b473f1d6b..01e5714b5 100644 --- a/5-appinfra/modules/htc-infra/agent/src/serve/pubsub_push.go +++ b/5-appinfra/modules/htc-infra/agent/src/serve/pubsub_push.go @@ -23,9 +23,10 @@ import ( "net/http" "os" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/serve/serve.go b/5-appinfra/modules/htc-infra/agent/src/serve/serve.go index 82e20271a..3ca779855 100644 --- a/5-appinfra/modules/htc-infra/agent/src/serve/serve.go +++ b/5-appinfra/modules/htc-infra/agent/src/serve/serve.go @@ -15,9 +15,10 @@ package serve import ( - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/test/generate.go b/5-appinfra/modules/htc-infra/agent/src/test/generate.go index 90d743641..d89979e95 100644 --- a/5-appinfra/modules/htc-infra/agent/src/test/generate.go +++ b/5-appinfra/modules/htc-infra/agent/src/test/generate.go @@ -15,8 +15,9 @@ package test import ( - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/test/grpc.go b/5-appinfra/modules/htc-infra/agent/src/test/grpc.go index 25533ad74..d25527b82 100644 --- a/5-appinfra/modules/htc-infra/agent/src/test/grpc.go +++ b/5-appinfra/modules/htc-infra/agent/src/test/grpc.go @@ -18,9 +18,10 @@ import ( "context" "fmt" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" "google.golang.org/protobuf/proto" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/test/pubsub.go b/5-appinfra/modules/htc-infra/agent/src/test/pubsub.go index ec15a2665..4b16d1f79 100644 --- a/5-appinfra/modules/htc-infra/agent/src/test/pubsub.go +++ b/5-appinfra/modules/htc-infra/agent/src/test/pubsub.go @@ -27,9 +27,10 @@ import ( "fmt" "time" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/5-appinfra/modules/htc-infra/agent/src/test/source.go b/5-appinfra/modules/htc-infra/agent/src/test/source.go index 7f20335f9..00c186430 100644 --- a/5-appinfra/modules/htc-infra/agent/src/test/source.go +++ b/5-appinfra/modules/htc-infra/agent/src/test/source.go @@ -20,9 +20,10 @@ import ( "sync/atomic" "time" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" diff --git a/5-appinfra/modules/htc-infra/apis.tf b/5-appinfra/modules/htc-infra/apis.tf index 6373fab6f..b3f97c00f 100644 --- a/5-appinfra/modules/htc-infra/apis.tf +++ b/5-appinfra/modules/htc-infra/apis.tf @@ -36,7 +36,7 @@ resource "google_project_iam_member" "team_roles_infra_project" { "roles/pubsub.viewer", "roles/pubsub.subscriber", "roles/monitoring.viewer", - "roles/privateca.admin" + "roles/privateca.certificateManager" ]) project = var.infra_project @@ -52,7 +52,7 @@ resource "google_project_iam_member" "team_roles_cluster_project" { "roles/pubsub.viewer", "roles/pubsub.subscriber", "roles/monitoring.viewer", - "roles/privateca.admin" + "roles/privateca.certificateManager" ]) project = var.cluster_project_id diff --git a/5-appinfra/modules/htc-infra/loadtest/go.mod b/5-appinfra/modules/htc-infra/loadtest/go.mod index 9c33ab42a..d69a52f2c 100644 --- a/5-appinfra/modules/htc-infra/loadtest/go.mod +++ b/5-appinfra/modules/htc-infra/loadtest/go.mod @@ -1,4 +1,4 @@ -module github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest +module github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/loadtest go 1.24 diff --git a/5-appinfra/modules/htc-infra/loadtest/request.proto b/5-appinfra/modules/htc-infra/loadtest/request.proto index f97f600fd..a42b4d2e2 100644 --- a/5-appinfra/modules/htc-infra/loadtest/request.proto +++ b/5-appinfra/modules/htc-infra/loadtest/request.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package main; -option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; +option go_package = "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/loadtest;main"; message LoadTask { diff --git a/5-appinfra/modules/htc-infra/loadtest/response.proto b/5-appinfra/modules/htc-infra/loadtest/response.proto index 5828dd25c..e1c903dee 100644 --- a/5-appinfra/modules/htc-infra/loadtest/response.proto +++ b/5-appinfra/modules/htc-infra/loadtest/response.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package main; -option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; +option go_package = "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/loadtest;main"; message LoadResult { diff --git a/5-appinfra/modules/htc-infra/loadtest/service.proto b/5-appinfra/modules/htc-infra/loadtest/service.proto index e66c04211..f5b6f7552 100644 --- a/5-appinfra/modules/htc-infra/loadtest/service.proto +++ b/5-appinfra/modules/htc-infra/loadtest/service.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package main; -option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; +option go_package = "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/loadtest;main"; import "request.proto"; import "response.proto"; diff --git a/5-appinfra/modules/htc-infra/main.tf b/5-appinfra/modules/htc-infra/main.tf index a8b8cdec6..6969200f7 100644 --- a/5-appinfra/modules/htc-infra/main.tf +++ b/5-appinfra/modules/htc-infra/main.tf @@ -74,6 +74,15 @@ module "agent" { } } +module "keda" { + source = "./modules/keda" + + project_id = var.admin_project + region = var.region + repository_region = var.region + repository_id = var.service_name +} + #----------------------------------------------------- # GKE Deployment #----------------------------------------------------- @@ -105,6 +114,9 @@ module "gke" { parallelstore_instances = local.parallelstore_instances vpc_name = var.network_name + keda_image = module.keda.keda_images.keda + keda_apiserver_image = module.keda.keda_images.keda-metrics-apiserver + depends_on = [ module.agent ] diff --git a/5-appinfra/modules/htc-infra/modules/artifact-registry/versions.tf b/5-appinfra/modules/htc-infra/modules/artifact-registry/versions.tf index 1eac1780d..4cde6343b 100644 --- a/5-appinfra/modules/htc-infra/modules/artifact-registry/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/artifact-registry/versions.tf @@ -33,7 +33,4 @@ terraform { version = ">= 3.7.2" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/modules/builder/versions.tf b/5-appinfra/modules/htc-infra/modules/builder/versions.tf index 1eac1780d..4cde6343b 100644 --- a/5-appinfra/modules/htc-infra/modules/builder/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/builder/versions.tf @@ -33,7 +33,4 @@ terraform { version = ">= 3.7.2" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/modules/keda/keda.tf b/5-appinfra/modules/htc-infra/modules/keda/keda.tf new file mode 100644 index 000000000..d5eef6846 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/keda/keda.tf @@ -0,0 +1,102 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + repository_prefix = "${var.repository_region}-docker.pkg.dev/${var.project_id}/${var.repository_id}" + + keda_version = "2.18.2" + + keda_target_operator = "${local.repository_prefix}/keda" + keda_target_adapter = "${local.repository_prefix}/keda-metrics-apiserver" +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +resource "google_storage_bucket" "cloudbuild_keda" { + project = var.project_id + location = var.region + name = "${var.project_id}-${var.region}-cloudbuild-keda-${random_string.suffix.id}" + uniform_bucket_level_access = true + force_destroy = true +} + +resource "null_resource" "mirror_keda_images" { + # Re-run this process if the KEDA version changes + triggers = { + version = local.keda_version + } + + provisioner "local-exec" { + command = <<-EOT + set -e + # Create a temporary directory for the build context + mkdir -p keda_mirror_build + mkdir -p keda_mirror_build_apiserver + + # --- BUILD KEDA OPERATOR --- + # Create a simple Dockerfile that pulls the external image + echo "FROM ghcr.io/kedacore/keda:${local.keda_version}" > keda_mirror_build/Dockerfile + + # Submit build to Cloud Build (Handles Pull -> Tag -> Push) + gcloud builds submit \ + --project ${var.project_id} \ + --region ${var.region} \ + --gcs-source-staging-dir gs://${google_storage_bucket.cloudbuild_keda.id}/source/keda-operator \ + --gcs-log-dir gs://${google_storage_bucket.cloudbuild_keda.id}/logs/keda-operator \ + --tag "${local.keda_target_operator}:${local.keda_version}" \ + --tag "${local.keda_target_operator}:latest" \ + keda_mirror_build + + # --- BUILD KEDA ADAPTER --- + # Update Dockerfile for the adapter image + echo "FROM ghcr.io/kedacore/keda-metrics-apiserver:${local.keda_version}" > keda_mirror_build_apiserver/Dockerfile + + # Submit build to Cloud Build + gcloud builds submit \ + --project ${var.project_id} \ + --region ${var.region} \ + --gcs-source-staging-dir gs://${google_storage_bucket.cloudbuild_keda.id}/source/keda-adapter \ + --gcs-log-dir gs://${google_storage_bucket.cloudbuild_keda.id}/logs/keda-adapter \ + --tag "${local.keda_target_adapter}:${local.keda_version}" \ + --tag "${local.keda_target_adapter}:latest" \ + keda_mirror_build_apiserver + + # Cleanup local temp directory + rm -rf keda_mirror_build + rm -rf keda_mirror_build_apiserver + EOT + } +} + +data "google_artifact_registry_docker_image" "keda_operator" { + depends_on = [null_resource.mirror_keda_images] + + project = var.project_id + location = var.repository_region + repository_id = var.repository_id + image_name = "keda" +} + +data "google_artifact_registry_docker_image" "keda_api_server" { + depends_on = [null_resource.mirror_keda_images] + + project = var.project_id + location = var.repository_region + repository_id = var.repository_id + image_name = "keda-metrics-apiserver" +} diff --git a/5-appinfra/modules/htc-infra/modules/keda/outputs.tf b/5-appinfra/modules/htc-infra/modules/keda/outputs.tf new file mode 100644 index 000000000..35566ac26 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/keda/outputs.tf @@ -0,0 +1,25 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "keda_images" { + description = "KEDA image URLs including SHA digests, ready for Binary Authorization" + value = { + keda = data.google_artifact_registry_docker_image.keda_operator.self_link + keda-metrics-apiserver = data.google_artifact_registry_docker_image.keda_api_server.self_link + } + + depends_on = [ + null_resource.mirror_keda_images + ] +} diff --git a/5-appinfra/modules/htc-infra/modules/keda/variables.tf b/5-appinfra/modules/htc-infra/modules/keda/variables.tf new file mode 100644 index 000000000..906f7c551 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/keda/variables.tf @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "The GCP project ID where resources will be created." +} + +variable "region" { + type = string + description = "The region of the build" +} + +variable "repository_region" { + type = string + description = "Artifacte Repository region" + +} + +variable "repository_id" { + type = string + description = "Artifact repository ID" +} diff --git a/5-appinfra/modules/htc-infra/modules/keda/versions.tf b/5-appinfra/modules/htc-infra/modules/keda/versions.tf new file mode 100644 index 000000000..0ec67b361 --- /dev/null +++ b/5-appinfra/modules/htc-infra/modules/keda/versions.tf @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.29.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.2.4" + } + local = { + source = "hashicorp/local" + version = ">= 2.5.2" + } + random = { + source = "hashicorp/random" + version = ">= 3.7.2" + } + } +} diff --git a/5-appinfra/modules/htc-infra/modules/kubectl/versions.tf b/5-appinfra/modules/htc-infra/modules/kubectl/versions.tf index d21257e22..30ffdd410 100644 --- a/5-appinfra/modules/htc-infra/modules/kubectl/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/kubectl/versions.tf @@ -25,7 +25,4 @@ terraform { version = ">= 6.29.0" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/modules/lustre/versions.tf b/5-appinfra/modules/htc-infra/modules/lustre/versions.tf index 1eac1780d..4cde6343b 100644 --- a/5-appinfra/modules/htc-infra/modules/lustre/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/lustre/versions.tf @@ -33,7 +33,4 @@ terraform { version = ">= 3.7.2" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/modules/parallelstore/versions.tf b/5-appinfra/modules/htc-infra/modules/parallelstore/versions.tf index 1eac1780d..4cde6343b 100644 --- a/5-appinfra/modules/htc-infra/modules/parallelstore/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/parallelstore/versions.tf @@ -33,7 +33,4 @@ terraform { version = ">= 3.7.2" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/modules/project/versions.tf b/5-appinfra/modules/htc-infra/modules/project/versions.tf index d21257e22..30ffdd410 100644 --- a/5-appinfra/modules/htc-infra/modules/project/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/project/versions.tf @@ -25,7 +25,4 @@ terraform { version = ">= 6.29.0" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/versions.tf b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/versions.tf index 13b507f25..d52db154d 100644 --- a/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/pubsub-subscriptions/versions.tf @@ -37,7 +37,4 @@ terraform { version = ">= 2.5.3" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/modules/quota/versions.tf b/5-appinfra/modules/htc-infra/modules/quota/versions.tf index d21257e22..30ffdd410 100644 --- a/5-appinfra/modules/htc-infra/modules/quota/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/quota/versions.tf @@ -25,7 +25,4 @@ terraform { version = ">= 6.29.0" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/modules/region-analysis/versions.tf b/5-appinfra/modules/htc-infra/modules/region-analysis/versions.tf index d21257e22..30ffdd410 100644 --- a/5-appinfra/modules/htc-infra/modules/region-analysis/versions.tf +++ b/5-appinfra/modules/htc-infra/modules/region-analysis/versions.tf @@ -25,7 +25,4 @@ terraform { version = ">= 6.29.0" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-agent-v1.0.0" - } } diff --git a/5-appinfra/modules/htc-infra/versions.tf b/5-appinfra/modules/htc-infra/versions.tf index fc108a017..8e7697fba 100644 --- a/5-appinfra/modules/htc-infra/versions.tf +++ b/5-appinfra/modules/htc-infra/versions.tf @@ -37,7 +37,4 @@ terraform { version = ">= 3.7.2" } } - provider_meta "google" { - module_name = "cloud-solutions/fsi-rdp-loadtest-v1.0.0" - } } diff --git a/examples/high-throughput-compute-standalone/QUICKSTART.md b/examples/high-throughput-compute-standalone/QUICKSTART.md index f1ecb4ae7..28d08a0ce 100644 --- a/examples/high-throughput-compute-standalone/QUICKSTART.md +++ b/examples/high-throughput-compute-standalone/QUICKSTART.md @@ -7,7 +7,7 @@ This is an example of running a loadtest library on Google Cloud infrastructure. It will run using GKE horizontal pod autoscaler orchestrated using Pub/Sub, and will also run on Cloud Run through BigQuery. For details on the loadtest library, see -its [README.md](https://github.com/GoogleCloudPlatform/risk-and-research-blueprints/blob/main/examples/risk/loadtest/src/README.md). The same techniques can be used to run any kind of +its [README.md](github.com/GoogleCloudPlatform/risk-and-research-blueprints/blob/main/examples/risk/loadtest/src/README.md). The same techniques can be used to run any kind of library that is exposing gRPC. Cloud Logging, Pub/Sub, Cloud Monitoring, BigQuery, and Looker Studio will all be used @@ -126,7 +126,7 @@ The link to the [subscription](https://console.cloud.google.com/cloudpubsub/subs will show you the health, statistics, and also allow you to pull messages (tasks). Pull some messages (do not Ack!) and see the messages appear. These messages -will be JSON, but adhering to the [protobuf request](https://github.com/GoogleCloudPlatform/risk-and-research-blueprints/blob/main/examples/risk/loadtest/src/request.proto) schema. +will be JSON, but adhering to the [protobuf request](github.com/GoogleCloudPlatform/risk-and-research-blueprints/blob/main/examples/risk/loadtest/src/request.proto) schema. ### Inspect the workers diff --git a/examples/high-throughput-compute-standalone/agent/src/go.mod b/examples/high-throughput-compute-standalone/agent/src/go.mod index 3b329820b..a7fe11689 100644 --- a/examples/high-throughput-compute-standalone/agent/src/go.mod +++ b/examples/high-throughput-compute-standalone/agent/src/go.mod @@ -1,4 +1,4 @@ -module github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent +module github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent go 1.24 toolchain go1.24.1 diff --git a/examples/high-throughput-compute-standalone/agent/src/main.go b/examples/high-throughput-compute-standalone/agent/src/main.go index 138a792d2..a2d46b707 100644 --- a/examples/high-throughput-compute-standalone/agent/src/main.go +++ b/examples/high-throughput-compute-standalone/agent/src/main.go @@ -24,10 +24,11 @@ import ( "syscall" "time" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/serve" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/test" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/serve" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/test" + "github.com/spf13/cobra" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/protoio/backend.go b/examples/high-throughput-compute-standalone/agent/src/protoio/backend.go index 734d01d5f..22a8a8ca0 100644 --- a/examples/high-throughput-compute-standalone/agent/src/protoio/backend.go +++ b/examples/high-throughput-compute-standalone/agent/src/protoio/backend.go @@ -24,8 +24,9 @@ import ( "log/slog" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/jhump/protoreflect/grpcreflect" "github.com/spf13/cobra" "google.golang.org/grpc" diff --git a/examples/high-throughput-compute-standalone/agent/src/protoio/io.go b/examples/high-throughput-compute-standalone/agent/src/protoio/io.go index d0638190d..29d358491 100644 --- a/examples/high-throughput-compute-standalone/agent/src/protoio/io.go +++ b/examples/high-throughput-compute-standalone/agent/src/protoio/io.go @@ -22,7 +22,8 @@ import ( "iter" "log/slog" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/protoio/io_test.go b/examples/high-throughput-compute-standalone/agent/src/protoio/io_test.go index c52e9529b..29842c25c 100644 --- a/examples/high-throughput-compute-standalone/agent/src/protoio/io_test.go +++ b/examples/high-throughput-compute-standalone/agent/src/protoio/io_test.go @@ -21,7 +21,7 @@ import ( "google.golang.org/protobuf/proto" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" ) // Command to compile the protobuf for tests: diff --git a/examples/high-throughput-compute-standalone/agent/src/protoio/test_service.proto b/examples/high-throughput-compute-standalone/agent/src/protoio/test_service.proto index 36657a536..947a53db8 100644 --- a/examples/high-throughput-compute-standalone/agent/src/protoio/test_service.proto +++ b/examples/high-throughput-compute-standalone/agent/src/protoio/test_service.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package main; -option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio"; +option go_package = "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio"; service TestService { diff --git a/examples/high-throughput-compute-standalone/agent/src/serve/bigquery_rdf.go b/examples/high-throughput-compute-standalone/agent/src/serve/bigquery_rdf.go index fc0c92c2a..15a3bc060 100644 --- a/examples/high-throughput-compute-standalone/agent/src/serve/bigquery_rdf.go +++ b/examples/high-throughput-compute-standalone/agent/src/serve/bigquery_rdf.go @@ -23,8 +23,9 @@ import ( "net/http" "os" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/serve/file.go b/examples/high-throughput-compute-standalone/agent/src/serve/file.go index 1a7b27f0b..be5fbdd70 100644 --- a/examples/high-throughput-compute-standalone/agent/src/serve/file.go +++ b/examples/high-throughput-compute-standalone/agent/src/serve/file.go @@ -15,8 +15,9 @@ package serve import ( - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" "google.golang.org/protobuf/proto" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/serve/pubsub.go b/examples/high-throughput-compute-standalone/agent/src/serve/pubsub.go index 49a5d4c22..5907f4b6a 100644 --- a/examples/high-throughput-compute-standalone/agent/src/serve/pubsub.go +++ b/examples/high-throughput-compute-standalone/agent/src/serve/pubsub.go @@ -20,10 +20,11 @@ import ( "log/slog" "time" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "cloud.google.com/go/pubsub" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" "github.com/spf13/cobra" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/serve/pubsub_push.go b/examples/high-throughput-compute-standalone/agent/src/serve/pubsub_push.go index b473f1d6b..01e5714b5 100644 --- a/examples/high-throughput-compute-standalone/agent/src/serve/pubsub_push.go +++ b/examples/high-throughput-compute-standalone/agent/src/serve/pubsub_push.go @@ -23,9 +23,10 @@ import ( "net/http" "os" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/serve/serve.go b/examples/high-throughput-compute-standalone/agent/src/serve/serve.go index 82e20271a..3ca779855 100644 --- a/examples/high-throughput-compute-standalone/agent/src/serve/serve.go +++ b/examples/high-throughput-compute-standalone/agent/src/serve/serve.go @@ -15,9 +15,10 @@ package serve import ( - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/test/generate.go b/examples/high-throughput-compute-standalone/agent/src/test/generate.go index 90d743641..d89979e95 100644 --- a/examples/high-throughput-compute-standalone/agent/src/test/generate.go +++ b/examples/high-throughput-compute-standalone/agent/src/test/generate.go @@ -15,8 +15,9 @@ package test import ( - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/test/grpc.go b/examples/high-throughput-compute-standalone/agent/src/test/grpc.go index 25533ad74..d25527b82 100644 --- a/examples/high-throughput-compute-standalone/agent/src/test/grpc.go +++ b/examples/high-throughput-compute-standalone/agent/src/test/grpc.go @@ -18,9 +18,10 @@ import ( "context" "fmt" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" "google.golang.org/protobuf/proto" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/test/pubsub.go b/examples/high-throughput-compute-standalone/agent/src/test/pubsub.go index ec15a2665..4b16d1f79 100644 --- a/examples/high-throughput-compute-standalone/agent/src/test/pubsub.go +++ b/examples/high-throughput-compute-standalone/agent/src/test/pubsub.go @@ -27,9 +27,10 @@ import ( "fmt" "time" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" ) diff --git a/examples/high-throughput-compute-standalone/agent/src/test/source.go b/examples/high-throughput-compute-standalone/agent/src/test/source.go index 7f20335f9..00c186430 100644 --- a/examples/high-throughput-compute-standalone/agent/src/test/source.go +++ b/examples/high-throughput-compute-standalone/agent/src/test/source.go @@ -20,9 +20,10 @@ import ( "sync/atomic" "time" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/gcp" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/protoio" - "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/agent/stats" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/gcp" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/protoio" + "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/agent/stats" + "github.com/spf13/cobra" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" diff --git a/examples/high-throughput-compute-standalone/modules/infrastructure/README.md b/examples/high-throughput-compute-standalone/modules/infrastructure/README.md index ec301cb57..d07dea323 100644 --- a/examples/high-throughput-compute-standalone/modules/infrastructure/README.md +++ b/examples/high-throughput-compute-standalone/modules/infrastructure/README.md @@ -22,7 +22,7 @@ This Terraform example demonstrates how to deploy a core infrastructure setup in 1. **Clone the Repository:** ```bash - git clone https://github.com/GoogleCloudPlatform/risk-and-research-blueprints.git + git clone github.com/GoogleCloudPlatform/risk-and-research-blueprints.git cd risk-and-research-blueprints/examples/infrastructure ``` diff --git a/examples/high-throughput-compute-standalone/src/go.mod b/examples/high-throughput-compute-standalone/src/go.mod index 9c33ab42a..d69a52f2c 100644 --- a/examples/high-throughput-compute-standalone/src/go.mod +++ b/examples/high-throughput-compute-standalone/src/go.mod @@ -1,4 +1,4 @@ -module github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest +module github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/loadtest go 1.24 diff --git a/examples/high-throughput-compute-standalone/src/request.proto b/examples/high-throughput-compute-standalone/src/request.proto index f97f600fd..a42b4d2e2 100644 --- a/examples/high-throughput-compute-standalone/src/request.proto +++ b/examples/high-throughput-compute-standalone/src/request.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package main; -option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; +option go_package = "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/loadtest;main"; message LoadTask { diff --git a/examples/high-throughput-compute-standalone/src/response.proto b/examples/high-throughput-compute-standalone/src/response.proto index 5828dd25c..e1c903dee 100644 --- a/examples/high-throughput-compute-standalone/src/response.proto +++ b/examples/high-throughput-compute-standalone/src/response.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package main; -option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; +option go_package = "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/loadtest;main"; message LoadResult { diff --git a/examples/high-throughput-compute-standalone/src/service.proto b/examples/high-throughput-compute-standalone/src/service.proto index e66c04211..f5b6f7552 100644 --- a/examples/high-throughput-compute-standalone/src/service.proto +++ b/examples/high-throughput-compute-standalone/src/service.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package main; -option go_package = "github.com/GoogleCloudPlatform/finance-research-risk-examples/examples/risk/loadtest;main"; +option go_package = "github.com/GoogleCloudPlatform/risk-and-research-blueprints/examples/risk/loadtest;main"; import "request.proto"; import "response.proto"; diff --git a/examples/hpc/README.md b/examples/hpc/README.md index 79b2fbd90..d0c6efb28 100644 --- a/examples/hpc/README.md +++ b/examples/hpc/README.md @@ -9,7 +9,7 @@ This document presents two use cases that demonstrate the utilization of `kueue` ## Use Case 1: HPC Financial Analysis Using Monte Carlo Simulations (Team: `hpc-team-b`) -This example is an adaptation from [Google Cloud Platform's Risk and Research Blueprints](https://github.com/GoogleCloudPlatform/risk-and-research-blueprints/tree/main/examples/research/monte-carlo). +This example is an adaptation from [Google Cloud Platform's Risk and Research Blueprints](github.com/GoogleCloudPlatform/risk-and-research-blueprints/tree/main/examples/research/monte-carlo). ### Requirements @@ -226,7 +226,7 @@ After uploading the data to the bucket, you may proceed. Follow the steps outlined in the following document, after the "Open the Vertex AI Workbench Notebook" section: -[Open the Vertex AI Workbench Notebook](https://github.com/GoogleCloudPlatform/risk-and-research-blueprints/tree/0e3134b8478f3ffaa12031d7fda3ac6b94e61b17/examples/research/monte-carlo#open-the-vertex-ai-workbench-notebook) +[Open the Vertex AI Workbench Notebook](github.com/GoogleCloudPlatform/risk-and-research-blueprints/tree/0e3134b8478f3ffaa12031d7fda3ac6b94e61b17/examples/research/monte-carlo#open-the-vertex-ai-workbench-notebook) **IMPORTANT**: Your Vertex AI Workbench Instance will be located on the application infrastructure project that was created on 4-appfactory. diff --git a/examples/htc/6-appsource/cloudbuild.yaml b/examples/htc/6-appsource/cloudbuild.yaml index 82de4edb3..cdbc62b71 100644 --- a/examples/htc/6-appsource/cloudbuild.yaml +++ b/examples/htc/6-appsource/cloudbuild.yaml @@ -23,9 +23,7 @@ steps: AGENT_DIGEST=$(gcloud artifacts docker images describe "$$IMAGE_URI" \ --format='get(image_summary.digest)') echo "Found digest for agent: $$AGENT_DIGEST" - # Store the digest in a file for subsequent steps echo "export AGENT_DIGEST=$$AGENT_DIGEST" >> /workspace/digests.env - # We write to /workspace/digests.env, which is shared across steps - id: 'get-loadtest-image-digest' name: 'gcr.io/cloud-builders/gcloud' @@ -37,20 +35,42 @@ steps: LOADTEST_DIGEST=$(gcloud artifacts docker images describe "$$IMAGE_URI" \ --format='get(image_summary.digest)') echo "Found digest for loadtest: $$LOADTEST_DIGEST" - # Append the digest to the same file echo "export LOADTEST_DIGEST=$$LOADTEST_DIGEST" >> /workspace/digests.env + - id: 'get-keda-image-digest' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: + - '-c' + - | + IMAGE_URI="$_CONTAINER_REGISTRY/keda:latest" + KEDA_DIGEST=$(gcloud artifacts docker images describe "$$IMAGE_URI" \ + --format='get(image_summary.digest)') + echo "Found digest for keda: $$KEDA_DIGEST" + echo "export KEDA_DIGEST=$$KEDA_DIGEST" >> /workspace/digests.env + + - id: 'get-keda-api-server-image-digest' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: + - '-c' + - | + IMAGE_URI="$_CONTAINER_REGISTRY/keda-metrics-apiserver:latest" + KEDA_API_SERVER_DIGEST=$(gcloud artifacts docker images describe "$$IMAGE_URI" \ + --format='get(image_summary.digest)') + echo "Found digest for keda api server: $$KEDA_API_SERVER_DIGEST" + echo "export KEDA_API_SERVER_DIGEST=$$KEDA_API_SERVER_DIGEST" >> /workspace/digests.env + - id: 'get-american-option-image-digest' name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' args: - '-c' - | - IMAGE_URI="$_CONTAINER_REGISTRY/american_option:latest" # Corrected from agent:latest + IMAGE_URI="$_CONTAINER_REGISTRY/american_option:latest" AMERICANOPTION_DIGEST=$(gcloud artifacts docker images describe "$$IMAGE_URI" \ --format='get(image_summary.digest)') echo "Found digest for american-option: $$AMERICANOPTION_DIGEST" - # Append the digest to the same file echo "export AMERICANOPTION_DIGEST=$$AMERICANOPTION_DIGEST" >> /workspace/digests.env - id: 'log-image-paths' @@ -59,7 +79,6 @@ steps: args: - '-c' - | - # Source the file to load the variables into this step's environment source /workspace/digests.env echo "Attesting agent image: $_CONTAINER_REGISTRY/agent@$$AGENT_DIGEST" echo "Attesting loadtest image: $_CONTAINER_REGISTRY/loadtest@$$LOADTEST_DIGEST" @@ -73,7 +92,6 @@ steps: args: - '-c' - | - # Source the file to load the variables into this step's environment source /workspace/digests.env /work/create_binauthz_attestation.sh \ '--artifact-url' \ @@ -113,6 +131,36 @@ steps: '--keyversion' \ "$_KMS_KEY_VERSION" + - id: 'create-keda-attestation' + name: '$_BINARY_AUTH_IMAGE' + entrypoint: 'bash' + args: + - '-c' + - | + source /workspace/digests.env + /work/create_binauthz_attestation.sh \ + '--artifact-url' \ + "$_CONTAINER_REGISTRY/keda@$$KEDA_DIGEST" \ + '--attestor' \ + "$_ATTESTOR_ID" \ + '--keyversion' \ + "$_KMS_KEY_VERSION" + + - id: 'create-keda-api-server-attestation' + name: '$_BINARY_AUTH_IMAGE' + entrypoint: 'bash' + args: + - '-c' + - | + source /workspace/digests.env + /work/create_binauthz_attestation.sh \ + '--artifact-url' \ + "$_CONTAINER_REGISTRY/keda-metrics-apiserver@$$KEDA_API_SERVER_DIGEST" \ + '--attestor' \ + "$_ATTESTOR_ID" \ + '--keyversion' \ + "$_KMS_KEY_VERSION" + options: logging: CLOUD_LOGGING_ONLY pool: diff --git a/examples/htc/README.md b/examples/htc/README.md index 060b8176f..1aab3ee34 100644 --- a/examples/htc/README.md +++ b/examples/htc/README.md @@ -7,7 +7,7 @@ This example shows how to deploy the HTC example using the infrastructure create This is an example of running a loadtest library on Google Cloud infrastructure. It will run using GKE horizontal pod autoscaler orchestrated using Pub/Sub. For details on the loadtest library, see -its [README.md](https://github.com/GoogleCloudPlatform/risk-and-research-blueprints/blob/main/examples/risk/loadtest/src/README.md). The same techniques can be used to run any kind of +its [README.md](github.com/GoogleCloudPlatform/risk-and-research-blueprints/blob/main/examples/risk/loadtest/src/README.md). The same techniques can be used to run any kind of library that is exposing gRPC. Cloud Logging, Pub/Sub, Cloud Monitoring, BigQuery, and Looker Studio will all be used @@ -130,6 +130,8 @@ The steps below assume that you are checked out on the same level as `terraform- echo htc_repository=$htc_repository export htc_statebucket=$(terraform output -json app-group | jq -r '.["htc.htc"]["app_cloudbuild_workspace_state_bucket_name"]' | sed 's/.*\///') echo htc_statebucket=$htc_statebucket + export htc_cb_sa=$(terraform output -json app-group | jq -r '.["htc.htc"]["app_cloudbuild_workspace_cloudbuild_sa_email"]' | sed 's/.*\///') + echo htc_cb_sa=$htc_cb_sa cd ../../../ ``` @@ -202,11 +204,30 @@ The steps below assume that you are checked out on the same level as `terraform- 1. Merge plan to production branch: - ```bash + ```bash git checkout -b production git push --set-upstream origin production ``` +##### Workspace group + +The HTC example requires the cloud build's service account created during the app infra stage to be added to the namespace's workspace group. + +1. Define your target group email + + ```bash + GROUP_EMAIL="example@groupemail.com" + ``` + +2. Execute the gcloud command + + ```bash + gcloud identity groups memberships add \ + --group-email="$GROUP_EMAIL" \ + --member-email="$htc_cb_sa" \ + --roles="MEMBER" + ``` + #### Application Source Code pipeline This section describes how to clone the empty application source repository, copy the example code into it, and push it to trigger the CI/CD pipeline. @@ -228,19 +249,19 @@ First, retrieve the names of the source code repository and its Google Cloud pro 1. Clone the empty repository created by the App Factory. (CSR Only) ```bash - gcloud source repos clone $app_source_repository --project=$app_source_project + gcloud source repos clone $app_source_repository --project=$app_source_project ``` 1. Clone the empty repository created by the App Factory. (CSR Only) ```bash - git clone git@github.com:/$app_source_repository.git + git clone git@github.com:/$app_source_repository.git ``` 1. Clone the empty repository created by the App Factory. (Gitlab Only) ```bash - git clone git@gitlab.com:/$app_source_repository.git + git clone git@gitlab.com:/$app_source_repository.git ``` 1. Copy and Commit the Example Source Code @@ -248,11 +269,11 @@ First, retrieve the names of the source code repository and its Google Cloud pro Define the path to the example source code, copy it into your new repository, and commit the files. ```bash - export APP_SOURCE_DIR_PATH=$(readlink -f ./terraform-google-enterprise-application/examples/htc/6-appsource) - cd $app_source_repository - cp -r $APP_SOURCE_DIR_PATH/* ./ - git add . - git commit -m "Add HTC application source code" + export APP_SOURCE_DIR_PATH=$(readlink -f ./terraform-google-enterprise-application/examples/htc/6-appsource) + cd $app_source_repository + cp -r $APP_SOURCE_DIR_PATH/* ./ + git add . + git commit -m "Add HTC application source code" ``` 1. Push to main to Trigger the Pipeline @@ -260,5 +281,5 @@ Define the path to the example source code, copy it into your new repository, an Push the code to the main branch. This will trigger the Cloud Build pipeline to build and deploy the HTC application. ```bash - git push --set-upstream origin main + git push --set-upstream origin main ``` From aefccb16ad480a888ee454fae52237d6d0c031ad Mon Sep 17 00:00:00 2001 From: marcelorobj Date: Mon, 19 Jan 2026 11:06:41 -0300 Subject: [PATCH 7/7] fix: addressing open pr comments --- .../htc-infra/american-option/README.md | 98 ------------------ .../modules/htc-infra/loadtest/README.md | 99 ------------------- 5-appinfra/modules/htc-infra/sql/README.md | 21 ---- 5-appinfra/modules/htc-infra/variables.tf | 62 ------------ .../5-appinfra/htc/envs/production/README.md | 1 - 5 files changed, 281 deletions(-) delete mode 100644 5-appinfra/modules/htc-infra/american-option/README.md delete mode 100644 5-appinfra/modules/htc-infra/loadtest/README.md delete mode 100644 5-appinfra/modules/htc-infra/sql/README.md diff --git a/5-appinfra/modules/htc-infra/american-option/README.md b/5-appinfra/modules/htc-infra/american-option/README.md deleted file mode 100644 index 07fefa737..000000000 --- a/5-appinfra/modules/htc-infra/american-option/README.md +++ /dev/null @@ -1,98 +0,0 @@ - -# American Option Example Library - -## Overview - -This is an example Python script using [Quantlib](https://www.quantlib.org/) in order -to calculate American Option prices. This is provided purely as an example and is not -intended to be used as-is for financial pricing. - -## Setup - -There is a provided [Dockerfile](Dockerfile) and a [setup script](setup.sh) to -configure a virtual environment and generate protobuf code. - -You can run `setup.sh` locally for development purposes and it can also be used -in the `Dockerfile`. - -## Running locally - -### Setup the virtual environment - -Run setup.sh: - -```sh -./setup.sh -``` - -### Generate test data - -Generate 1000 example tasks: - -```sh -.venv/bin/python3 main.py gentasks --count 1000 tasks.jsonl -``` - -### With the file directly - -Directly run pricer with the test data: - -```sh -.venv/bin/python3 main.py load tasks.jsonl -``` - -### With gRPC - -Run the pricing engine locally on the default port 2002: - -```sh -.venv/bin/python3 main.py serve -``` - -In a separate command prompt, [grpcurl](https://github.com/fullstorydev/grpcurl) with the test data to try it out: -to generate JSON test data for the pricing library. This will only send in the -first request. - -```sh -head -n 1 tasks.jsonl | grpcurl -d @ -plaintext localhost:2002 main.PricingService/CalcPrices -``` - -## Running in a container - -### Build the container - -```sh -docker build -t pricer . -``` - -### Generate test data - -Generate 1000 example tasks: - -```sh -docker run pricer gentasks --count 1000 - > tasks.jsonl -``` - -### With the file directly - -Directly run pricer with the test data: - -```sh -docker run -v $PWD:/data pricer load /data/tasks.jsonl -``` - -### With gRPC - -Run the pricing engine locally on the default port 2002: - -```sh -docker run -p 2002:2002 pricer serve -``` - -In a separate command prompt, [grpcurl](https://github.com/fullstorydev/grpcurl) with the test data to try it out: -to generate JSON test data for the pricing library. This will only send in the -first request. - -```sh -head -n 1 tasks.jsonl | grpcurl -d @ -plaintext localhost:2002 main.PricingService/CalcPrices -``` diff --git a/5-appinfra/modules/htc-infra/loadtest/README.md b/5-appinfra/modules/htc-infra/loadtest/README.md deleted file mode 100644 index 140abbc24..000000000 --- a/5-appinfra/modules/htc-infra/loadtest/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# Load Test for HTC - -## Overview - -This is a general purpose gRPC load test. It supports compute, reading/writing data, -initialization, and produces JSON formatted stats. - -## Compilation - -This can be compiled using `go generate` and `go build`. The following packages -are required for building: `protoc-gen-go`, `protoc-gen-go-grpc`, and -`golang-google-protobuf-dev`. There is a provided [Dockerfile](Dockerfile). - -## Running locally - -### Building - -```sh -go generate -go build -``` - -### Generate test data - -Generate 1000 example tasks: - -```sh -./loadtest gentasks --count 1000 tasks.jsonl -``` - -### With the file directly - -Directly run loadtest with the test data: - -```sh -./loadtest load tasks.jsonl -``` - -You will see both logging (on stderr) and the generated output (on stdout). - -### With gRPC - -Run the pricing engine locally on the default port 2002: - -```sh -./loadtest serve -``` - -In a separate command prompt, [grpcurl](https://github.com/fullstorydev/grpcurl) with the test data to try it out: -to generate JSON test data for the pricing library. This will only send in the -first request. - -```sh -head -n 1 tasks.jsonl | grpcurl -d @ -plaintext localhost:2002 main.PricingService/CalcPrices -``` - -NOTE: You can use the --logJSON to produce JSON formatted logs which includes some -performance statistics. These can be directly queried with distributed jobs using -Google Cloud Logging with [Log Analytics](https://cloud.google.com/logging/docs/analyze/query-and-view). - -## Running in a container - -### Build the container - -```sh -docker build -t loadtest . -``` - -### Generate test data - -Generate 1000 example tasks: - -```sh -docker run loadtest gentasks --count 1000 - > tasks.jsonl -``` - -### With the file directly - -Directly run loadtest with the test data: - -```sh -docker run -v $PWD:/data loadtest load /data/tasks.jsonl -``` - -### With gRPC - -Run the loadtest engine locally on the default port 2002: - -```sh -docker run -p 2002:2002 loadtest serve -``` - -In a separate command prompt, [grpcurl](https://github.com/fullstorydev/grpcurl) with the test data to try it out: -to generate JSON test data for the pricing library. This will only send in the -first request. - -```sh -head -n 1 tasks.jsonl | grpcurl -d @ -plaintext localhost:2002 main.LoadTestService/RunLibrary -``` diff --git a/5-appinfra/modules/htc-infra/sql/README.md b/5-appinfra/modules/htc-infra/sql/README.md deleted file mode 100644 index ce40dd9f5..000000000 --- a/5-appinfra/modules/htc-infra/sql/README.md +++ /dev/null @@ -1,21 +0,0 @@ - -Interesting query: - -SELECT - start_time, - jobId, - initId, - TIMESTAMP_DIFF(last_process_time, first_process_time, SECOND) as run_secs, - last_process_time AS last_time, - remaining_tasks AS queue, - CAST(SAFE_DIVIDE(100*task_jobs,remaining_tasks+task_jobs) AS STRING FORMAT '999.9') AS perc_done, - init_jobs, - CAST(SAFE_DIVIDE(task_compute_secs, task_jobs) AS STRING FORMAT '999999.9') AS init_cpu_avg, - CAST(SAFE_DIVIDE(init_io_secs_total, init_jobs) AS STRING FORMAT '999999.9') AS init_io_avg, - task_jobs, - CAST(SAFE_DIVIDE(task_compute_secs, task_jobs) AS STRING FORMAT '999999.9') AS task_cpu_avg, - CAST(SAFE_DIVIDE(task_io_secs_total, task_jobs) AS STRING FORMAT '999999.9') AS task_io_avg, -from - `fsi-scratch-18.workload.messages_summary` -ORDER BY - start_time DESC diff --git a/5-appinfra/modules/htc-infra/variables.tf b/5-appinfra/modules/htc-infra/variables.tf index a24f0f897..739cf7dc9 100644 --- a/5-appinfra/modules/htc-infra/variables.tf +++ b/5-appinfra/modules/htc-infra/variables.tf @@ -36,40 +36,6 @@ variable "pubsub_exactly_once" { default = true } -# variable "request_topic" { -# description = "Request topic for tasks" -# type = string -# default = "request" -# } - -# variable "request_subscription" { -# description = "Request subscription for tasks" -# type = string -# default = "request_sub" -# } - -# variable "response_topic" { -# description = "Response topic for tasks" -# type = string -# default = "response" -# } - -# variable "response_subscription" { -# description = "Response subscription for tasks" -# type = string -# default = "response_sub" -# } - -#----------------------------------------------------- -# BigQuery Configuration -#----------------------------------------------------- - -# variable "dataset_id" { -# description = "BigQuery dataset in the project to create the tables" -# type = string -# default = "pubsub_msgs" -# } - #----------------------------------------------------- # Quota Configuration #----------------------------------------------------- @@ -86,24 +52,6 @@ variable "quota_contact_email" { default = "" } -# variable "scaled_control_plane" { -# description = "Deploy a larger initial nodepool to ensure larger control plane nodes are provisioned" -# type = bool -# default = false -# } - -# variable "cluster_max_cpus" { -# description = "Maximum CPU cores in cluster autoscaling resource limits" -# type = number -# default = 10000 -# } - -# variable "cluster_max_memory" { -# description = "Maximum memory (in GB) in cluster autoscaling resource limits" -# type = number -# default = 80000 -# } - #----------------------------------------------------- # Storage Configuration #----------------------------------------------------- @@ -112,22 +60,12 @@ variable "storage_type" { description = "The type of storage system to deploy (PARALLELSTORE, LUSTRE, or null for none)" type = string default = null - - # validation { - # condition = var.storage_type == null || contains(["PARALLELSTORE", "LUSTRE"], var.storage_type) - # error_message = "The storage_type must be null, PARALLELSTORE, or LUSTRE." - # } } variable "storage_capacity_gib" { description = "Capacity in GiB for the selected storage system (Parallelstore or Lustre)" type = number default = null - - # validation { - # condition = var.storage_capacity_gib > 0 - # error_message = "Storage capacity must be a positive number or null." - # } } variable "storage_locations" { diff --git a/examples/htc/5-appinfra/htc/envs/production/README.md b/examples/htc/5-appinfra/htc/envs/production/README.md index b127bd7f3..8e8da7db4 100644 --- a/examples/htc/5-appinfra/htc/envs/production/README.md +++ b/examples/htc/5-appinfra/htc/envs/production/README.md @@ -1,4 +1,3 @@ - # Load Test for HTC ## Overview